{"item_id":"senseval2.d000.s000.t000","gold_sense_keys":["art%1:09:00::"],"candidates":[{"index":1,"sense_key":"art%1:04:00::","synset_id":"art.n.02"},{"index":2,"sense_key":"art%1:10:00::","synset_id":"artwork.n.01"},{"index":3,"sense_key":"art%1:09:00::","synset_id":"art.n.03"},{"index":4,"sense_key":"art%1:06:00::","synset_id":"art.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"art%1:09:00::","call_ids":["senseval2.d000.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"art%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.003595,"input_uncached_usd":0.0019450000000000001,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8154510830063373}
{"item_id":"senseval2.d000.s000.t002","gold_sense_keys":["peculiar%5:00:00:characteristic:00","peculiar%5:00:00:specific:00"],"candidates":[{"index":1,"sense_key":"peculiar%5:00:00:strange:00","synset_id":"curious.s.01"},{"index":2,"sense_key":"peculiar%5:00:00:characteristic:00","synset_id":"peculiar.s.04"},{"index":3,"sense_key":"peculiar%5:00:00:specific:00","synset_id":"particular.s.01"},{"index":4,"sense_key":"peculiar%5:00:00:unusual:00","synset_id":"peculiar.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"peculiar%5:00:00:specific:00","call_ids":["senseval2.d000.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"peculiar%5:00:00:specific:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":435,"reasoning_output_tokens":419},"cost":{"currency":"USD","total_usd":0.01515,"input_uncached_usd":0.0021000000000000003,"input_cached_usd":0.0,"output_usd":0.01305,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.201991541893221}
{"item_id":"senseval2.d000.s000.t003","gold_sense_keys":["english%1:18:00::"],"candidates":[{"index":1,"sense_key":"english%1:18:00::","synset_id":"english.n.02"},{"index":2,"sense_key":"english%1:09:00::","synset_id":"english.n.03"},{"index":3,"sense_key":"english%1:11:00::","synset_id":"english.n.04"},{"index":4,"sense_key":"english%1:10:00::","synset_id":"english.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"english%1:18:00::","call_ids":["senseval2.d000.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"english%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":333,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.0033150000000000002,"input_uncached_usd":0.001665,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9244874579599127}
{"item_id":"senseval2.d000.s000.t004","gold_sense_keys":["most%3:00:02::"],"candidates":[{"index":1,"sense_key":"most%3:00:02::","synset_id":"most.a.01"},{"index":2,"sense_key":"most%3:00:01::","synset_id":"most.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"most%3:00:02::","call_ids":["senseval2.d000.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"most%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":313,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.003935,"input_uncached_usd":0.0015650000000000002,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7337924590101466}
{"item_id":"senseval2.d000.s000.t005","gold_sense_keys":["english%3:01:00::"],"candidates":[{"index":1,"sense_key":"english%3:01:01::","synset_id":"english.a.02"},{"index":2,"sense_key":"english%3:01:00::","synset_id":"english.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"english%3:01:00::","call_ids":["senseval2.d000.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"english%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":255,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.004365,"input_uncached_usd":0.001275,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5690310419304296}
{"item_id":"senseval2.d000.s000.t006","gold_sense_keys":["peculiarity%1:07:02::","peculiarity%1:09:00::"],"candidates":[{"index":1,"sense_key":"peculiarity%1:09:00::","synset_id":"peculiarity.n.01"},{"index":2,"sense_key":"peculiarity%1:06:00::","synset_id":"curio.n.01"},{"index":3,"sense_key":"peculiarity%1:07:02::","synset_id":"peculiarity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"peculiarity%1:09:00::","call_ids":["senseval2.d000.s000.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"peculiarity%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":300,"cached_input_tokens":0,"output_tokens":429,"reasoning_output_tokens":413},"cost":{"currency":"USD","total_usd":0.014369999999999999,"input_uncached_usd":0.0015,"input_cached_usd":0.0,"output_usd":0.01287,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.739684624946676}
{"item_id":"senseval2.d000.s000.t007","gold_sense_keys":["unintelligible%5:00:00:incomprehensible:00"],"candidates":[{"index":1,"sense_key":"unintelligible%5:00:00:incomprehensible:00","synset_id":"opaque.s.01"},{"index":2,"sense_key":"unintelligible%3:00:00::","synset_id":"unintelligible.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"unintelligible%5:00:00:incomprehensible:00","call_ids":["senseval2.d000.s000.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"unintelligible%5:00:00:incomprehensible:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":270,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.00366,"input_uncached_usd":0.00135,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2240141249494627}
{"item_id":"senseval2.d000.s000.t008","gold_sense_keys":["rest%1:24:00::"],"candidates":[{"index":1,"sense_key":"rest%1:28:00::","synset_id":"respite.n.04"},{"index":2,"sense_key":"rest%1:04:00::","synset_id":"rest.n.02"},{"index":3,"sense_key":"rest%1:06:00::","synset_id":"rest.n.06"},{"index":4,"sense_key":"rest%1:26:00::","synset_id":"rest.n.04"},{"index":5,"sense_key":"rest%1:24:00::","synset_id":"remainder.n.01"},{"index":6,"sense_key":"rest%1:26:01::","synset_id":"rest.n.05"},{"index":7,"sense_key":"rest%1:10:00::","synset_id":"rest.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"rest%1:24:00::","call_ids":["senseval2.d000.s000.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"rest%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.00445,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.215906332945451}
{"item_id":"senseval2.d000.s000.t009","gold_sense_keys":["world%1:14:02::"],"candidates":[{"index":1,"sense_key":"world%1:14:01::","synset_id":"world.n.02"},{"index":2,"sense_key":"world%1:17:02::","synset_id":"world.n.06"},{"index":3,"sense_key":"world%1:14:02::","synset_id":"populace.n.01"},{"index":4,"sense_key":"world%1:05:00::","synset_id":"world.n.08"},{"index":5,"sense_key":"world%1:17:01::","synset_id":"universe.n.01"},{"index":6,"sense_key":"world%1:09:01::","synset_id":"worldly_concern.n.01"},{"index":7,"sense_key":"world%1:09:00::","synset_id":"world.n.03"},{"index":8,"sense_key":"world%1:17:00::","synset_id":"earth.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"world%1:05:00::","call_ids":["senseval2.d000.s000.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"world%1:05:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":4229,"reasoning_output_tokens":4165},"cost":{"currency":"USD","total_usd":0.12979500000000002,"input_uncached_usd":0.002925,"input_cached_usd":0.0,"output_usd":0.12687,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":72.36133420804981}
{"item_id":"senseval2.d000.s003.t000","gold_sense_keys":["scene%1:09:00::"],"candidates":[{"index":1,"sense_key":"scene%1:06:01::","synset_id":"scene.n.08"},{"index":2,"sense_key":"scene%1:26:01::","synset_id":"fit.n.01"},{"index":3,"sense_key":"scene%1:11:00::","synset_id":"scene.n.02"},{"index":4,"sense_key":"scene%1:10:01::","synset_id":"scene.n.04"},{"index":5,"sense_key":"scene%1:15:00::","synset_id":"scene.n.01"},{"index":6,"sense_key":"scene%1:06:00::","synset_id":"scenery.n.01"},{"index":7,"sense_key":"scene%1:15:01::","synset_id":"setting.n.01"},{"index":8,"sense_key":"scene%1:26:00::","synset_id":"picture.n.04"},{"index":9,"sense_key":"scene%1:09:00::","synset_id":"view.n.02"},{"index":10,"sense_key":"scene%1:10:00::","synset_id":"scene.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"scene%1:09:00::","call_ids":["senseval2.d000.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"scene%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":682,"cached_input_tokens":0,"output_tokens":396,"reasoning_output_tokens":380},"cost":{"currency":"USD","total_usd":0.01529,"input_uncached_usd":0.0034100000000000003,"input_cached_usd":0.0,"output_usd":0.01188,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.167811999912374}
{"item_id":"senseval2.d000.s003.t001","gold_sense_keys":["evoke%2:32:01::"],"candidates":[{"index":1,"sense_key":"evoke%2:37:00::","synset_id":"arouse.v.01"},{"index":2,"sense_key":"evoke%2:36:01::","synset_id":"educe.v.01"},{"index":3,"sense_key":"evoke%2:36:02::","synset_id":"raise.v.07"},{"index":4,"sense_key":"evoke%2:32:01::","synset_id":"suggest.v.05"},{"index":5,"sense_key":"evoke%2:36:00::","synset_id":"provoke.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"evoke%2:32:01::","call_ids":["senseval2.d000.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"evoke%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":101},"cost":{"currency":"USD","total_usd":0.00619,"input_uncached_usd":0.00268,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.311729458044283}
{"item_id":"senseval2.d000.s003.t004","gold_sense_keys":["lovely%5:00:00:beautiful:00"],"candidates":[{"index":1,"sense_key":"lovely%5:00:00:beautiful:00","synset_id":"lovely.s.01"},{"index":2,"sense_key":"lovely%5:00:00:lovable:00","synset_id":"adorable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lovely%5:00:00:beautiful:00","call_ids":["senseval2.d000.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lovely%5:00:00:beautiful:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":337,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.0040550000000000004,"input_uncached_usd":0.001685,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0461253749672323}
{"item_id":"senseval2.d000.s003.t005","gold_sense_keys":["ancient%5:00:00:old:02"],"candidates":[{"index":1,"sense_key":"ancient%5:00:00:old:02","synset_id":"ancient.s.02"},{"index":2,"sense_key":"ancient%5:00:00:past:00","synset_id":"ancient.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ancient%5:00:00:old:02","call_ids":["senseval2.d000.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ancient%5:00:00:old:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":359,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.003715,"input_uncached_usd":0.0017950000000000002,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.257162583991885}
{"item_id":"senseval2.d000.s003.t006","gold_sense_keys":["stone%1:06:00::"],"candidates":[{"index":1,"sense_key":"stone%1:27:01::","synset_id":"gem.n.02"},{"index":2,"sense_key":"stone%1:06:00::","synset_id":"stone.n.02"},{"index":3,"sense_key":"stone%1:20:00::","synset_id":"stone.n.06"},{"index":4,"sense_key":"stone%1:23:00::","synset_id":"stone.n.05"},{"index":5,"sense_key":"stone%1:18:05::","synset_id":"stone.n.07"},{"index":6,"sense_key":"stone%1:18:03::","synset_id":"stone.n.09"},{"index":7,"sense_key":"stone%1:17:00::","synset_id":"rock.n.01"},{"index":8,"sense_key":"stone%1:18:04::","synset_id":"stone.n.08"},{"index":9,"sense_key":"stone%1:07:00::","synset_id":"stone.n.13"},{"index":10,"sense_key":"stone%1:18:00::","synset_id":"stone.n.12"},{"index":11,"sense_key":"stone%1:27:00::","synset_id":"rock.n.02"},{"index":12,"sense_key":"stone%1:18:01::","synset_id":"stone.n.11"},{"index":13,"sense_key":"stone%1:18:02::","synset_id":"stone.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"stone%1:27:00::","call_ids":["senseval2.d000.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"stone%1:27:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":885,"cached_input_tokens":0,"output_tokens":422,"reasoning_output_tokens":406},"cost":{"currency":"USD","total_usd":0.017085,"input_uncached_usd":0.004425,"input_cached_usd":0.0,"output_usd":0.012660000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.46860912500415}
{"item_id":"senseval2.d000.s003.t007","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":2,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":3,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s003.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.003855,"input_uncached_usd":0.002085,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.258238042006269}
{"item_id":"senseval2.d000.s003.t008","gold_sense_keys":["stand%2:42:03::"],"candidates":[{"index":1,"sense_key":"stand%2:42:01::","synset_id":"stand.v.09"},{"index":2,"sense_key":"stand%2:35:01::","synset_id":"stand.v.10"},{"index":3,"sense_key":"stand%2:35:10::","synset_id":"stand.v.12"},{"index":4,"sense_key":"stand%2:42:06::","synset_id":"stand.v.07"},{"index":5,"sense_key":"stand%2:42:00::","synset_id":"stand.v.04"},{"index":6,"sense_key":"stand%2:42:04::","synset_id":"stand.v.02"},{"index":7,"sense_key":"stand%2:33:00::","synset_id":"resist.v.04"},{"index":8,"sense_key":"stand%2:42:03::","synset_id":"stand.v.03"},{"index":9,"sense_key":"stand%2:31:01::","synset_id":"stand.v.06"},{"index":10,"sense_key":"stand%2:35:00::","synset_id":"stand.v.01"},{"index":11,"sense_key":"stand%2:31:00::","synset_id":"digest.v.03"},{"index":12,"sense_key":"stand%2:42:02::","synset_id":"stand.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"stand%2:42:03::","call_ids":["senseval2.d000.s003.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"stand%2:42:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.005235,"input_uncached_usd":0.003615,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1729284999892116}
{"item_id":"senseval2.d000.s003.t009","gold_sense_keys":["field%1:15:00::"],"candidates":[{"index":1,"sense_key":"field%1:14:01::","synset_id":"field.n.12"},{"index":2,"sense_key":"field%1:26:00::","synset_id":"sphere.n.01"},{"index":3,"sense_key":"field%1:06:00::","synset_id":"airfield.n.01"},{"index":4,"sense_key":"field%1:17:00::","synset_id":"plain.n.01"},{"index":5,"sense_key":"field%1:09:00::","synset_id":"discipline.n.01"},{"index":6,"sense_key":"field%1:15:05::","synset_id":"field.n.14"},{"index":7,"sense_key":"field%1:14:00::","synset_id":"field.n.10"},{"index":8,"sense_key":"field%1:15:00::","synset_id":"field.n.01"},{"index":9,"sense_key":"field%1:15:01::","synset_id":"field.n.03"},{"index":10,"sense_key":"field%1:15:03::","synset_id":"field.n.11"},{"index":11,"sense_key":"field%1:04:00::","synset_id":"field.n.06"},{"index":12,"sense_key":"field%1:19:00::","synset_id":"field.n.05"},{"index":13,"sense_key":"field%1:15:02::","synset_id":"playing_field.n.02"},{"index":14,"sense_key":"field%1:14:02::","synset_id":"field.n.13"},{"index":15,"sense_key":"field%1:14:03::","synset_id":"field.n.15"},{"index":16,"sense_key":"field%1:09:01::","synset_id":"field.n.16"},{"index":17,"sense_key":"field%1:15:04::","synset_id":"battlefield.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"field%1:15:00::","call_ids":["senseval2.d000.s003.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"field%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1038,"cached_input_tokens":0,"output_tokens":143,"reasoning_output_tokens":127},"cost":{"currency":"USD","total_usd":0.00948,"input_uncached_usd":0.00519,"input_cached_usd":0.0,"output_usd":0.00429,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.126715667080134}
{"item_id":"senseval2.d000.s003.t010","gold_sense_keys":["sound%1:07:00::"],"candidates":[{"index":1,"sense_key":"sound%1:17:01::","synset_id":"strait.n.01"},{"index":2,"sense_key":"sound%1:17:00::","synset_id":"sound.n.08"},{"index":3,"sense_key":"sound%1:10:00::","synset_id":"audio.n.01"},{"index":4,"sense_key":"sound%1:11:00::","synset_id":"sound.n.04"},{"index":5,"sense_key":"sound%1:10:01::","synset_id":"phone.n.02"},{"index":6,"sense_key":"sound%1:09:00::","synset_id":"sound.n.02"},{"index":7,"sense_key":"sound%1:07:00::","synset_id":"sound.n.01"},{"index":8,"sense_key":"sound%1:19:00::","synset_id":"sound.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"sound%1:07:00::","call_ids":["senseval2.d000.s003.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"sound%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.004545,"input_uncached_usd":0.0028950000000000004,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.895522458013147}
{"item_id":"senseval2.d000.s003.t011","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":2,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":3,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":4,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":5,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"},{"index":6,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":7,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":8,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":9,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":10,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s003.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019305,"input_uncached_usd":0.003465,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.22164799994789}
{"item_id":"senseval2.d000.s003.t012","gold_sense_keys":["cascade%2:38:00::"],"candidates":[{"index":1,"sense_key":"cascade%2:35:00::","synset_id":"cascade.v.02"},{"index":2,"sense_key":"cascade%2:38:00::","synset_id":"cascade.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cascade%2:38:00::","call_ids":["senseval2.d000.s003.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cascade%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":337,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.0036049999999999997,"input_uncached_usd":0.001685,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2587816659361124}
{"item_id":"senseval2.d000.s003.t013","gold_sense_keys":["tower%1:06:00::"],"candidates":[{"index":1,"sense_key":"tower%1:25:00::","synset_id":"column.n.04"},{"index":2,"sense_key":"tower%1:06:00::","synset_id":"tower.n.01"},{"index":3,"sense_key":"tower%1:06:01::","synset_id":"tugboat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tower%1:06:00::","call_ids":["senseval2.d000.s003.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tower%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.003745,"input_uncached_usd":0.0019450000000000001,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0781917499843985}
{"item_id":"senseval2.d000.s003.t014","gold_sense_keys":["call%2:32:05::"],"candidates":[{"index":1,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":2,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":3,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":4,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":5,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":6,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":7,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":8,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":9,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":10,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":11,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":12,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":13,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":14,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":15,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":16,"sense_key":"call%2:29:05::","synset_id":"call.v.28"},{"index":17,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":18,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":19,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":20,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":21,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":22,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":23,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":24,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":25,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":26,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":27,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":28,"sense_key":"call%2:32:14::","synset_id":"call.v.24"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"call%2:32:05::","call_ids":["senseval2.d000.s003.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"call%2:32:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1443,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.023055,"input_uncached_usd":0.007215000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.559653458069079}
{"item_id":"senseval2.d000.s003.t015","gold_sense_keys":["faithful%1:14:00::"],"candidates":[{"index":1,"sense_key":"faithful%1:14:00::","synset_id":"congregation.n.01"},{"index":2,"sense_key":"faithful%1:14:01::","synset_id":"faithful.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"faithful%1:14:00::","call_ids":["senseval2.d000.s003.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"faithful%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":331,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.003995,"input_uncached_usd":0.0016550000000000002,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6395531250163913}
{"item_id":"senseval2.d000.s003.t016","gold_sense_keys":["evensong%1:10:00::"],"candidates":[{"index":1,"sense_key":"evensong%1:28:00::","synset_id":"vespers.n.01"},{"index":2,"sense_key":"evensong%1:10:00::","synset_id":"evening_prayer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"evensong%1:10:00::","call_ids":["senseval2.d000.s003.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"evensong%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":357,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":126},"cost":{"currency":"USD","total_usd":0.006045,"input_uncached_usd":0.0017850000000000001,"input_cached_usd":0.0,"output_usd":0.00426,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.3851222089724615}
{"item_id":"senseval2.d000.s004.t001","gold_sense_keys":["stop%2:38:00::"],"candidates":[{"index":1,"sense_key":"stop%2:30:12::","synset_id":"hold_on.v.02"},{"index":2,"sense_key":"stop%2:41:00::","synset_id":"stop.v.03"},{"index":3,"sense_key":"stop%2:38:01::","synset_id":"stop.v.05"},{"index":4,"sense_key":"stop%2:33:00::","synset_id":"check.v.18"},{"index":5,"sense_key":"stop%2:30:05::","synset_id":"break.v.10"},{"index":6,"sense_key":"stop%2:38:02::","synset_id":"stop.v.04"},{"index":7,"sense_key":"stop%2:42:13::","synset_id":"end.v.01"},{"index":8,"sense_key":"stop%2:38:00::","synset_id":"stop.v.01"},{"index":9,"sense_key":"stop%2:35:12::","synset_id":"barricade.v.01"},{"index":10,"sense_key":"stop%2:35:00::","synset_id":"intercept.v.01"},{"index":11,"sense_key":"stop%2:42:00::","synset_id":"discontinue.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"stop%2:38:00::","call_ids":["senseval2.d000.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"stop%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":135},"cost":{"currency":"USD","total_usd":0.008585,"input_uncached_usd":0.0040550000000000004,"input_cached_usd":0.0,"output_usd":0.00453,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.178002542001195}
{"item_id":"senseval2.d000.s004.t003","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":2,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.003995,"input_uncached_usd":0.002225,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9144563750596717}
{"item_id":"senseval2.d000.s004.t004","gold_sense_keys":["door%1:06:00::","door%1:06:01::"],"candidates":[{"index":1,"sense_key":"door%1:06:04::","synset_id":"door.n.04"},{"index":2,"sense_key":"door%1:06:00::","synset_id":"door.n.01"},{"index":3,"sense_key":"door%1:06:01::","synset_id":"doorway.n.01"},{"index":4,"sense_key":"door%1:06:03::","synset_id":"door.n.05"},{"index":5,"sense_key":"door%1:07:00::","synset_id":"door.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"door%1:06:01::","call_ids":["senseval2.d000.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"door%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":279,"reasoning_output_tokens":263},"cost":{"currency":"USD","total_usd":0.011054999999999999,"input_uncached_usd":0.0026850000000000003,"input_cached_usd":0.0,"output_usd":0.00837,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.59010670799762}
{"item_id":"senseval2.d000.s004.t005","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":2,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"},{"index":3,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":4,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":5,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"member%1:18:00::","call_ids":["senseval2.d000.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"member%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":73},"cost":{"currency":"USD","total_usd":0.00536,"input_uncached_usd":0.00272,"input_cached_usd":0.0,"output_usd":0.00264,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.165130584035069}
{"item_id":"senseval2.d000.s004.t006","gold_sense_keys":["here%4:02:00::"],"candidates":[{"index":1,"sense_key":"here%4:02:02::","synset_id":"here.r.02"},{"index":2,"sense_key":"here%4:02:00::","synset_id":"here.r.01"},{"index":3,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":4,"sense_key":"here%4:02:04::","synset_id":"here.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"here%4:02:00::","call_ids":["senseval2.d000.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"here%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.005025,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4111172090051696}
{"item_id":"senseval2.d000.s004.t007","gold_sense_keys":["always%4:02:00::"],"candidates":[{"index":1,"sense_key":"always%4:02:01::","synset_id":"always.r.05"},{"index":2,"sense_key":"always%4:02:02::","synset_id":"always.r.04"},{"index":3,"sense_key":"always%4:02:00::","synset_id":"always.r.01"},{"index":4,"sense_key":"always%4:02:03::","synset_id":"constantly.r.02"},{"index":5,"sense_key":"always%4:02:04::","synset_id":"constantly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"always%4:02:00::","call_ids":["senseval2.d000.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"always%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018395,"input_uncached_usd":0.0025550000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.130339875002392}
{"item_id":"senseval2.d000.s005.t000","gold_sense_keys":["tower%1:06:00::"],"candidates":[{"index":1,"sense_key":"tower%1:06:01::","synset_id":"tugboat.n.01"},{"index":2,"sense_key":"tower%1:25:00::","synset_id":"column.n.04"},{"index":3,"sense_key":"tower%1:06:00::","synset_id":"tower.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"tower%1:06:00::","call_ids":["senseval2.d000.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"tower%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.003625,"input_uncached_usd":0.002275,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.388207250041887}
{"item_id":"senseval2.d000.s005.t001","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":2,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":3,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":4,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":5,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":6,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":7,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":8,"sense_key":"man%1:05:00::","synset_id":"world.n.08"},{"index":9,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":10,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":11,"sense_key":"man%1:18:08::","synset_id":"man.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval2.d000.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":868,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.00743,"input_uncached_usd":0.00434,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.892353333067149}
{"item_id":"senseval2.d000.s005.t002","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"},{"index":2,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":3,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":4,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.0049250000000000006,"input_uncached_usd":0.002645,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.920005000079982}
{"item_id":"senseval2.d000.s005.t003","gold_sense_keys":["pull%2:35:04::"],"candidates":[{"index":1,"sense_key":"pull%2:35:00::","synset_id":"pull.v.01"},{"index":2,"sense_key":"pull%2:38:03::","synset_id":"draw.v.05"},{"index":3,"sense_key":"pull%2:35:10::","synset_id":"extract.v.01"},{"index":4,"sense_key":"pull%2:41:00::","synset_id":"perpetrate.v.01"},{"index":5,"sense_key":"pull%2:38:13::","synset_id":"pull.v.09"},{"index":6,"sense_key":"pull%2:35:03::","synset_id":"pull.v.13"},{"index":7,"sense_key":"pull%2:30:11::","synset_id":"pull.v.17"},{"index":8,"sense_key":"pull%2:35:05::","synset_id":"pluck.v.05"},{"index":9,"sense_key":"pull%2:38:00::","synset_id":"pull.v.07"},{"index":10,"sense_key":"pull%2:35:04::","synset_id":"pull.v.04"},{"index":11,"sense_key":"pull%2:35:01::","synset_id":"rend.v.01"},{"index":12,"sense_key":"pull%2:38:01::","synset_id":"pull.v.03"},{"index":13,"sense_key":"pull%2:29:00::","synset_id":"pull.v.08"},{"index":14,"sense_key":"pull%2:38:02::","synset_id":"pull.v.10"},{"index":15,"sense_key":"pull%2:33:13::","synset_id":"pull.v.16"},{"index":16,"sense_key":"pull%2:35:02::","synset_id":"attract.v.01"},{"index":17,"sense_key":"pull%2:38:04::","synset_id":"pull.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"pull%2:35:04::","call_ids":["senseval2.d000.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"pull%2:35:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1086,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.00897,"input_uncached_usd":0.005430000000000001,"input_cached_usd":0.0,"output_usd":0.00354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.89389379194472}
{"item_id":"senseval2.d000.s005.t005","gold_sense_keys":["rope%1:06:00::"],"candidates":[{"index":1,"sense_key":"rope%1:06:00::","synset_id":"rope.n.01"},{"index":2,"sense_key":"rope%1:06:01::","synset_id":"r-2.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rope%1:06:00::","call_ids":["senseval2.d000.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rope%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":399,"cached_input_tokens":0,"output_tokens":41,"reasoning_output_tokens":26},"cost":{"currency":"USD","total_usd":0.0032250000000000004,"input_uncached_usd":0.0019950000000000002,"input_cached_usd":0.0,"output_usd":0.00123,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9374151249649003}
{"item_id":"senseval2.d000.s005.t007","gold_sense_keys":["same%3:00:02::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"},{"index":3,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":4,"sense_key":"same%3:00:04::","synset_id":"like.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"same%3:00:02::","call_ids":["senseval2.d000.s005.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"same%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.004535000000000001,"input_uncached_usd":0.002465,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0387077500345185}
{"item_id":"senseval2.d000.s005.t008","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":2,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":3,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"},{"index":4,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":5,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":6,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":7,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":8,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":9,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":10,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s005.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":759,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.0060750000000000005,"input_uncached_usd":0.003795,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4650111669907346}
{"item_id":"senseval2.d000.s005.t009","gold_sense_keys":["first%4:02:02::"],"candidates":[{"index":1,"sense_key":"first%4:02:00::","synset_id":"first.r.01"},{"index":2,"sense_key":"first%4:02:03::","synset_id":"first.r.03"},{"index":3,"sense_key":"first%4:02:01::","synset_id":"foremost.r.01"},{"index":4,"sense_key":"first%4:02:02::","synset_id":"first.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"first%4:02:02::","call_ids":["senseval2.d000.s005.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"first%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.00499,"input_uncached_usd":0.0024400000000000003,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.93876395910047}
{"item_id":"senseval2.d000.s005.t010","gold_sense_keys":["sound%2:39:00::","sound%2:39:03::"],"candidates":[{"index":1,"sense_key":"sound%2:39:02::","synset_id":"sound.v.04"},{"index":2,"sense_key":"sound%2:39:06::","synset_id":"sound.v.01"},{"index":3,"sense_key":"sound%2:39:00::","synset_id":"sound.v.02"},{"index":4,"sense_key":"sound%2:30:00::","synset_id":"fathom.v.02"},{"index":5,"sense_key":"sound%2:39:03::","synset_id":"sound.v.03"},{"index":6,"sense_key":"sound%2:39:01::","synset_id":"sound.v.06"},{"index":7,"sense_key":"sound%2:32:00::","synset_id":"voice.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"sound%2:39:00::","call_ids":["senseval2.d000.s005.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"sound%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":1557,"reasoning_output_tokens":1529},"cost":{"currency":"USD","total_usd":0.04951,"input_uncached_usd":0.0028000000000000004,"input_cached_usd":0.0,"output_usd":0.04671,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":32.50090141699184}
{"item_id":"senseval2.d000.s005.t011","gold_sense_keys":["here%4:02:00::"],"candidates":[{"index":1,"sense_key":"here%4:02:04::","synset_id":"here.r.04"},{"index":2,"sense_key":"here%4:02:02::","synset_id":"here.r.02"},{"index":3,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":4,"sense_key":"here%4:02:00::","synset_id":"here.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"here%4:02:00::","call_ids":["senseval2.d000.s005.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"here%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.005185,"input_uncached_usd":0.002425,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3368919159984216}
{"item_id":"senseval2.d000.s006.t001","gold_sense_keys":["discordant%3:00:00::"],"candidates":[{"index":1,"sense_key":"discordant%5:00:00:inharmonious:00","synset_id":"discordant.s.01"},{"index":2,"sense_key":"discordant%3:00:00::","synset_id":"discordant.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"discordant%3:00:00::","call_ids":["senseval2.d000.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"discordant%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017925,"input_uncached_usd":0.002085,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.628708167001605}
{"item_id":"senseval2.d000.s006.t003","gold_sense_keys":["note%1:07:00::"],"candidates":[{"index":1,"sense_key":"note%1:07:00::","synset_id":"note.n.05"},{"index":2,"sense_key":"note%1:26:00::","synset_id":"eminence.n.01"},{"index":3,"sense_key":"note%1:10:02::","synset_id":"note.n.07"},{"index":4,"sense_key":"note%1:10:05::","synset_id":"note.n.04"},{"index":5,"sense_key":"note%1:21:01::","synset_id":"bill.n.03"},{"index":6,"sense_key":"note%1:10:04::","synset_id":"note.n.03"},{"index":7,"sense_key":"note%1:21:00::","synset_id":"note.n.09"},{"index":8,"sense_key":"note%1:10:03::","synset_id":"note.n.02"},{"index":9,"sense_key":"note%1:10:00::","synset_id":"note.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"note%1:07:00::","call_ids":["senseval2.d000.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"note%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":196,"reasoning_output_tokens":181},"cost":{"currency":"USD","total_usd":0.00965,"input_uncached_usd":0.0037700000000000003,"input_cached_usd":0.0,"output_usd":0.00588,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.936055166996084}
{"item_id":"senseval2.d000.s006.t005","gold_sense_keys":["enjoy%2:34:00::","enjoy%2:37:00::","enjoy%2:37:01::"],"candidates":[{"index":1,"sense_key":"enjoy%2:37:00::","synset_id":"enjoy.v.01"},{"index":2,"sense_key":"enjoy%2:39:00::","synset_id":"enjoy.v.04"},{"index":3,"sense_key":"enjoy%2:34:01::","synset_id":"enjoy.v.02"},{"index":4,"sense_key":"enjoy%2:34:00::","synset_id":"delight.v.02"},{"index":5,"sense_key":"enjoy%2:37:01::","synset_id":"love.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"enjoy%2:37:00::","call_ids":["senseval2.d000.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"enjoy%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":827,"reasoning_output_tokens":805},"cost":{"currency":"USD","total_usd":0.027395000000000003,"input_uncached_usd":0.002585,"input_cached_usd":0.0,"output_usd":0.024810000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":16.316711249994114}
{"item_id":"senseval2.d000.s006.t007","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"},{"index":2,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":3,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":4,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":5,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":6,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":7,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":8,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":9,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":10,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bell%1:11:00::","call_ids":["senseval2.d000.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bell%1:11:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019695,"input_uncached_usd":0.0038550000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.814166874974035}
{"item_id":"senseval2.d000.s006.t008","gold_sense_keys":["cool%3:00:01::"],"candidates":[{"index":1,"sense_key":"cool%3:00:01::","synset_id":"cool.a.01"},{"index":2,"sense_key":"cool%3:00:03::","synset_id":"cool.a.03"},{"index":3,"sense_key":"cool%3:00:02::","synset_id":"cool.a.04"},{"index":4,"sense_key":"cool%5:00:00:fashionable:00","synset_id":"cool.s.03"},{"index":5,"sense_key":"cool%5:00:00:unqualified:02","synset_id":"cool.s.02"},{"index":6,"sense_key":"cool%5:00:00:composed:00","synset_id":"cool.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cool%3:00:01::","call_ids":["senseval2.d000.s006.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cool%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.0044150000000000005,"input_uncached_usd":0.0030350000000000004,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0048547090264037}
{"item_id":"senseval2.d000.s006.t010","gold_sense_keys":["evening%1:28:00::"],"candidates":[{"index":1,"sense_key":"evening%1:28:00::","synset_id":"evening.n.01"},{"index":2,"sense_key":"evening%1:28:02::","synset_id":"evening.n.03"},{"index":3,"sense_key":"evening%1:28:01::","synset_id":"evening.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evening%1:28:00::","call_ids":["senseval2.d000.s006.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evening%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":118},"cost":{"currency":"USD","total_usd":0.006365000000000001,"input_uncached_usd":0.0023750000000000004,"input_cached_usd":0.0,"output_usd":0.0039900000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.7241955830249935}
{"item_id":"senseval2.d000.s007.t000","gold_sense_keys":["most%3:00:02::"],"candidates":[{"index":1,"sense_key":"most%3:00:01::","synset_id":"most.a.02"},{"index":2,"sense_key":"most%3:00:02::","synset_id":"most.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"most%3:00:02::","call_ids":["senseval2.d000.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"most%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.005105,"input_uncached_usd":0.0023450000000000003,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.057375625008717}
{"item_id":"senseval2.d000.s007.t001","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":2,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d000.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.004880000000000001,"input_uncached_usd":0.0026000000000000003,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4227037499658763}
{"item_id":"senseval2.d000.s007.t002","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":2,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":3,"sense_key":"church%1:14:01::","synset_id":"church.n.04"},{"index":4,"sense_key":"church%1:06:00::","synset_id":"church.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.005285,"input_uncached_usd":0.0024950000000000003,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4275515830377117}
{"item_id":"senseval2.d000.s007.t004","gold_sense_keys":["set%1:14:00::"],"candidates":[{"index":1,"sense_key":"set%1:09:00::","synset_id":"set.n.12"},{"index":2,"sense_key":"set%1:06:00::","synset_id":"stage_set.n.01"},{"index":3,"sense_key":"set%1:06:01::","synset_id":"set.n.13"},{"index":4,"sense_key":"set%1:14:02::","synset_id":"set.n.02"},{"index":5,"sense_key":"set%1:18:00::","synset_id":"set.n.10"},{"index":6,"sense_key":"set%1:22:00::","synset_id":"hardening.n.02"},{"index":7,"sense_key":"set%1:28:00::","synset_id":"set.n.08"},{"index":8,"sense_key":"set%1:04:01::","synset_id":"set.n.07"},{"index":9,"sense_key":"set%1:11:00::","synset_id":"set.n.11"},{"index":10,"sense_key":"set%1:14:00::","synset_id":"set.n.01"},{"index":11,"sense_key":"set%1:04:00::","synset_id":"set.n.03"},{"index":12,"sense_key":"set%1:14:01::","synset_id":"set.n.05"},{"index":13,"sense_key":"set%1:09:01::","synset_id":"bent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"set%1:14:00::","call_ids":["senseval2.d000.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"set%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":904,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.006320000000000001,"input_uncached_usd":0.004520000000000001,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1223544579697773}
{"item_id":"senseval2.d000.s007.t005","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":2,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":3,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":4,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":7,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":8,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":9,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.005845,"input_uncached_usd":0.0038650000000000004,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2913607080699876}
{"item_id":"senseval2.d000.s007.t006","gold_sense_keys":["once%4:02:01::"],"candidates":[{"index":1,"sense_key":"once%4:02:01::","synset_id":"once.r.03"},{"index":2,"sense_key":"once%4:02:00::","synset_id":"once.r.01"},{"index":3,"sense_key":"once%4:02:02::","synset_id":"once.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"once%4:02:01::","call_ids":["senseval2.d000.s007.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"once%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.00405,"input_uncached_usd":0.00231,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0149594160029665}
{"item_id":"senseval2.d000.s007.t009","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":3,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s007.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.003795,"input_uncached_usd":0.002415,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.009323542006314}
{"item_id":"senseval2.d000.s007.t011","gold_sense_keys":["sunday%1:28:00::"],"candidates":[{"index":1,"sense_key":"sunday%1:28:00::","synset_id":"sunday.n.01"},{"index":2,"sense_key":"sunday%1:18:00::","synset_id":"sunday.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sunday%1:28:00::","call_ids":["senseval2.d000.s007.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sunday%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.00367,"input_uncached_usd":0.00214,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6687955419765785}
{"item_id":"senseval2.d000.s007.t012","gold_sense_keys":["morning%1:28:00::"],"candidates":[{"index":1,"sense_key":"morning%1:11:00::","synset_id":"dawn.n.02"},{"index":2,"sense_key":"morning%1:28:00::","synset_id":"morning.n.01"},{"index":3,"sense_key":"morning%1:28:01::","synset_id":"dawn.n.01"},{"index":4,"sense_key":"morning%1:10:00::","synset_id":"good_morning.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"morning%1:28:00::","call_ids":["senseval2.d000.s007.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"morning%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.004465,"input_uncached_usd":0.002575,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2305896669859067}
{"item_id":"senseval2.d000.s007.t013","gold_sense_keys":["evening%1:28:00::"],"candidates":[{"index":1,"sense_key":"evening%1:28:01::","synset_id":"evening.n.02"},{"index":2,"sense_key":"evening%1:28:00::","synset_id":"evening.n.01"},{"index":3,"sense_key":"evening%1:28:02::","synset_id":"evening.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"evening%1:28:00::","call_ids":["senseval2.d000.s007.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"evening%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":111},"cost":{"currency":"USD","total_usd":0.0062050000000000004,"input_uncached_usd":0.002395,"input_cached_usd":0.0,"output_usd":0.00381,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.280564125045203}
{"item_id":"senseval2.d000.s007.t014","gold_sense_keys":["service%1:04:01::"],"candidates":[{"index":1,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":2,"sense_key":"service%1:04:06::","synset_id":"service.n.15"},{"index":3,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":4,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":5,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":6,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":7,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":8,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":9,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":10,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":11,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":12,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":13,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":14,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":15,"sense_key":"service%1:04:01::","synset_id":"service.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"service%1:04:01::","call_ids":["senseval2.d000.s007.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"service%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":925,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.006425,"input_uncached_usd":0.004625000000000001,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.160357665969059}
{"item_id":"senseval2.d000.s008.t000","gold_sense_keys":["now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":2,"sense_key":"now%4:02:06::","synset_id":"now.r.07"},{"index":3,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":4,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":5,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":6,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":7,"sense_key":"now%4:02:02::","synset_id":"now.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"now%4:02:00::","call_ids":["senseval2.d000.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"now%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019110000000000002,"input_uncached_usd":0.0032700000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.884022291982546}
{"item_id":"senseval2.d000.s008.t001","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":2,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":3,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:05::","synset_id":"only.r.07"},{"index":6,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":7,"sense_key":"only%4:02:03::","synset_id":"only.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"only%4:02:02::","call_ids":["senseval2.d000.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"only%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":791,"reasoning_output_tokens":769},"cost":{"currency":"USD","total_usd":0.026860000000000002,"input_uncached_usd":0.0031300000000000004,"input_cached_usd":0.0,"output_usd":0.02373,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":15.526311499997973}
{"item_id":"senseval2.d000.s008.t002","gold_sense_keys":["local%3:01:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:00:02::","synset_id":"local.a.03"},{"index":3,"sense_key":"local%3:01:01::","synset_id":"local.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval2.d000.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"local%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.004275,"input_uncached_usd":0.0023250000000000002,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1346275419928133}
{"item_id":"senseval2.d000.s008.t003","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.003945000000000001,"input_uncached_usd":0.002415,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.000053625088185}
{"item_id":"senseval2.d000.s008.t004","gold_sense_keys":["remain%2:42:05::"],"candidates":[{"index":1,"sense_key":"remain%2:42:01::","synset_id":"persist.v.03"},{"index":2,"sense_key":"remain%2:42:05::","synset_id":"remain.v.03"},{"index":3,"sense_key":"remain%2:30:00::","synset_id":"stay.v.01"},{"index":4,"sense_key":"remain%2:42:03::","synset_id":"stay.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"remain%2:42:05::","call_ids":["senseval2.d000.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"remain%2:42:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":225,"reasoning_output_tokens":210},"cost":{"currency":"USD","total_usd":0.009545,"input_uncached_usd":0.002795,"input_cached_usd":0.0,"output_usd":0.00675,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.245981000014581}
{"item_id":"senseval2.d000.s009.t000","gold_sense_keys":["here%4:02:00::"],"candidates":[{"index":1,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":2,"sense_key":"here%4:02:02::","synset_id":"here.r.02"},{"index":3,"sense_key":"here%4:02:04::","synset_id":"here.r.04"},{"index":4,"sense_key":"here%4:02:00::","synset_id":"here.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"here%4:02:00::","call_ids":["senseval2.d000.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"here%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":98},"cost":{"currency":"USD","total_usd":0.00594,"input_uncached_usd":0.00255,"input_cached_usd":0.0,"output_usd":0.0033900000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7809605420334265}
{"item_id":"senseval2.d000.s009.t001","gold_sense_keys":["today%1:28:01::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"today%1:28:01::","call_ids":["senseval2.d000.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"today%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":114},"cost":{"currency":"USD","total_usd":0.00604,"input_uncached_usd":0.00217,"input_cached_usd":0.0,"output_usd":0.00387,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9570298329927027}
{"item_id":"senseval2.d000.s009.t002","gold_sense_keys":["live%2:42:08::"],"candidates":[{"index":1,"sense_key":"live%2:31:00::","synset_id":"know.v.05"},{"index":2,"sense_key":"live%2:42:04::","synset_id":"survive.v.01"},{"index":3,"sense_key":"live%2:42:07::","synset_id":"live.v.07"},{"index":4,"sense_key":"live%2:42:01::","synset_id":"exist.v.02"},{"index":5,"sense_key":"live%2:42:08::","synset_id":"populate.v.01"},{"index":6,"sense_key":"live%2:42:00::","synset_id":"be.v.11"},{"index":7,"sense_key":"live%2:42:06::","synset_id":"live.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"live%2:42:08::","call_ids":["senseval2.d000.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"live%2:42:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.005275,"input_uncached_usd":0.0035350000000000004,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1578848339850083}
{"item_id":"senseval2.d000.s010.t001","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:09:00::","synset_id":"group.n.03"},{"index":3,"sense_key":"group%1:27:00::","synset_id":"group.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["senseval2.d000.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.0039000000000000003,"input_uncached_usd":0.0024000000000000002,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.946341749979183}
{"item_id":"senseval2.d000.s010.t002","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":2,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":3,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"},{"index":4,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.00397,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0282112499698997}
{"item_id":"senseval2.d000.s010.t003","gold_sense_keys":["include%2:42:00::"],"candidates":[{"index":1,"sense_key":"include%2:31:00::","synset_id":"include.v.02"},{"index":2,"sense_key":"include%2:41:03::","synset_id":"admit.v.03"},{"index":3,"sense_key":"include%2:42:00::","synset_id":"include.v.01"},{"index":4,"sense_key":"include%2:30:00::","synset_id":"include.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"include%2:42:00::","call_ids":["senseval2.d000.s010.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"include%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.00622,"input_uncached_usd":0.0027400000000000002,"input_cached_usd":0.0,"output_usd":0.00348,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9507793749216944}
{"item_id":"senseval2.d000.s010.t006","gold_sense_keys":["training%1:04:00::"],"candidates":[{"index":1,"sense_key":"training%1:07:00::","synset_id":"education.n.05"},{"index":2,"sense_key":"training%1:04:00::","synset_id":"training.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"training%1:04:00::","call_ids":["senseval2.d000.s010.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"training%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.004005,"input_uncached_usd":0.0021750000000000003,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1283012919593602}
{"item_id":"senseval2.d000.s010.t007","gold_sense_keys":["drive%2:38:00::"],"candidates":[{"index":1,"sense_key":"drive%2:42:00::","synset_id":"drive.v.12"},{"index":2,"sense_key":"drive%2:35:06::","synset_id":"drive.v.19"},{"index":3,"sense_key":"drive%2:35:03::","synset_id":"drive.v.17"},{"index":4,"sense_key":"drive%2:35:01::","synset_id":"drive.v.09"},{"index":5,"sense_key":"drive%2:38:11::","synset_id":"drive.v.16"},{"index":6,"sense_key":"drive%2:35:07::","synset_id":"drive.v.08"},{"index":7,"sense_key":"drive%2:41:02::","synset_id":"drive.v.05"},{"index":8,"sense_key":"drive%2:33:01::","synset_id":"drive.v.21"},{"index":9,"sense_key":"drive%2:35:00::","synset_id":"force.v.06"},{"index":10,"sense_key":"drive%2:36:00::","synset_id":"drive.v.07"},{"index":11,"sense_key":"drive%2:41:13::","synset_id":"drive.v.13"},{"index":12,"sense_key":"drive%2:38:10::","synset_id":"drive.v.15"},{"index":13,"sense_key":"drive%2:33:00::","synset_id":"drive.v.22"},{"index":14,"sense_key":"drive%2:35:11::","synset_id":"repel.v.01"},{"index":15,"sense_key":"drive%2:38:01::","synset_id":"drive.v.01"},{"index":16,"sense_key":"drive%2:32:00::","synset_id":"drive.v.11"},{"index":17,"sense_key":"drive%2:38:09::","synset_id":"drive.v.14"},{"index":18,"sense_key":"drive%2:34:07::","synset_id":"drive.v.20"},{"index":19,"sense_key":"drive%2:41:00::","synset_id":"tug.v.02"},{"index":20,"sense_key":"drive%2:38:00::","synset_id":"drive.v.02"},{"index":21,"sense_key":"drive%2:38:02::","synset_id":"drive.v.03"},{"index":22,"sense_key":"drive%2:35:02::","synset_id":"drive.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":20,"chosen_sense_key":"drive%2:38:00::","call_ids":["senseval2.d000.s010.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":20,"predicted_sense_key":"drive%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1223,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":125},"cost":{"currency":"USD","total_usd":0.010315000000000001,"input_uncached_usd":0.006115000000000001,"input_cached_usd":0.0,"output_usd":0.0042,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.081206709030084}
{"item_id":"senseval2.d000.s010.t008","gold_sense_keys":["sunday%1:28:00::"],"candidates":[{"index":1,"sense_key":"sunday%1:28:00::","synset_id":"sunday.n.01"},{"index":2,"sense_key":"sunday%1:18:00::","synset_id":"sunday.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sunday%1:28:00::","call_ids":["senseval2.d000.s010.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sunday%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":40,"reasoning_output_tokens":25},"cost":{"currency":"USD","total_usd":0.0034200000000000003,"input_uncached_usd":0.00222,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9996932920766994}
{"item_id":"senseval2.d000.s010.t009","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:01::","synset_id":"church.n.04"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:00::","synset_id":"church.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s010.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.005090000000000001,"input_uncached_usd":0.0025700000000000002,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6564841660438105}
{"item_id":"senseval2.d000.s010.t010","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":2,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":3,"sense_key":"church%1:14:01::","synset_id":"church.n.04"},{"index":4,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s010.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.00554,"input_uncached_usd":0.0025700000000000002,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.005705291056074}
{"item_id":"senseval2.d000.s010.t011","gold_sense_keys":["effort%1:04:00::"],"candidates":[{"index":1,"sense_key":"effort%1:04:00::","synset_id":"attempt.n.01"},{"index":2,"sense_key":"effort%1:04:03::","synset_id":"campaign.n.02"},{"index":3,"sense_key":"effort%1:04:01::","synset_id":"effort.n.02"},{"index":4,"sense_key":"effort%1:04:02::","synset_id":"feat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"effort%1:04:00::","call_ids":["senseval2.d000.s010.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"effort%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":311,"reasoning_output_tokens":295},"cost":{"currency":"USD","total_usd":0.012225,"input_uncached_usd":0.0028950000000000004,"input_cached_usd":0.0,"output_usd":0.00933,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.3542211659951136}
{"item_id":"senseval2.d000.s010.t012","gold_sense_keys":["keep%2:42:00::"],"candidates":[{"index":1,"sense_key":"keep%2:40:00::","synset_id":"keep.v.03"},{"index":2,"sense_key":"keep%2:42:03::","synset_id":"keep.v.12"},{"index":3,"sense_key":"keep%2:31:00::","synset_id":"observe.v.08"},{"index":4,"sense_key":"keep%2:41:02::","synset_id":"retain.v.02"},{"index":5,"sense_key":"keep%2:42:07::","synset_id":"continue.v.01"},{"index":6,"sense_key":"keep%2:41:03::","synset_id":"observe.v.09"},{"index":7,"sense_key":"keep%2:41:00::","synset_id":"restrain.v.01"},{"index":8,"sense_key":"keep%2:35:10::","synset_id":"keep.v.21"},{"index":9,"sense_key":"keep%2:40:09::","synset_id":"keep_open.v.01"},{"index":10,"sense_key":"keep%2:42:02::","synset_id":"keep.v.15"},{"index":11,"sense_key":"keep%2:41:01::","synset_id":"prevent.v.02"},{"index":12,"sense_key":"keep%2:41:05::","synset_id":"observe.v.06"},{"index":13,"sense_key":"keep%2:32:00::","synset_id":"keep.v.08"},{"index":14,"sense_key":"keep%2:40:10::","synset_id":"keep.v.20"},{"index":15,"sense_key":"keep%2:42:12::","synset_id":"keep.v.16"},{"index":16,"sense_key":"keep%2:34:00::","synset_id":"sustain.v.04"},{"index":17,"sense_key":"keep%2:30:10::","synset_id":"preserve.v.04"},{"index":18,"sense_key":"keep%2:42:00::","synset_id":"keep.v.01"},{"index":19,"sense_key":"keep%2:40:12::","synset_id":"keep.v.19"},{"index":20,"sense_key":"keep%2:40:01::","synset_id":"keep.v.07"},{"index":21,"sense_key":"keep%2:42:01::","synset_id":"keep.v.09"},{"index":22,"sense_key":"keep%2:40:13::","synset_id":"keep.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":18,"chosen_sense_key":"keep%2:42:00::","call_ids":["senseval2.d000.s010.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":18,"predicted_sense_key":"keep%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1293,"cached_input_tokens":0,"output_tokens":172,"reasoning_output_tokens":156},"cost":{"currency":"USD","total_usd":0.011625,"input_uncached_usd":0.006465,"input_cached_usd":0.0,"output_usd":0.0051600000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.9090764169814065}
{"item_id":"senseval2.d000.s010.t013","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":2,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s010.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":790,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.00704,"input_uncached_usd":0.00395,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.751409791992046}
{"item_id":"senseval2.d000.s010.t014","gold_sense_keys":["sound%2:39:00::","sound%2:39:03::"],"candidates":[{"index":1,"sense_key":"sound%2:39:03::","synset_id":"sound.v.03"},{"index":2,"sense_key":"sound%2:32:00::","synset_id":"voice.v.02"},{"index":3,"sense_key":"sound%2:30:00::","synset_id":"fathom.v.02"},{"index":4,"sense_key":"sound%2:39:00::","synset_id":"sound.v.02"},{"index":5,"sense_key":"sound%2:39:02::","synset_id":"sound.v.04"},{"index":6,"sense_key":"sound%2:39:06::","synset_id":"sound.v.01"},{"index":7,"sense_key":"sound%2:39:01::","synset_id":"sound.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sound%2:39:03::","call_ids":["senseval2.d000.s010.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sound%2:39:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018765,"input_uncached_usd":0.002955,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.679016791051254}
{"item_id":"senseval2.d000.s011.t000","gold_sense_keys":["even%4:02:00::"],"candidates":[{"index":1,"sense_key":"even%4:02:03::","synset_id":"even.r.02"},{"index":2,"sense_key":"even%4:02:01::","synset_id":"even.r.03"},{"index":3,"sense_key":"even%4:02:00::","synset_id":"even.r.01"},{"index":4,"sense_key":"even%4:02:04::","synset_id":"even.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"even%4:02:00::","call_ids":["senseval2.d000.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"even%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":523,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.0052250000000000005,"input_uncached_usd":0.002615,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9464493750128895}
{"item_id":"senseval2.d000.s011.t001","gold_sense_keys":["service%1:04:01::"],"candidates":[{"index":1,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":2,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":3,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":4,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":5,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":6,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":7,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":8,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":9,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":10,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":11,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":12,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":13,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":14,"sense_key":"service%1:04:06::","synset_id":"service.n.15"},{"index":15,"sense_key":"service%1:18:00::","synset_id":"service.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"service%1:04:01::","call_ids":["senseval2.d000.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"service%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":937,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.0065450000000000005,"input_uncached_usd":0.004685000000000001,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0886334580136463}
{"item_id":"senseval2.d000.s011.t002","gold_sense_keys":["tower%1:06:00::"],"candidates":[{"index":1,"sense_key":"tower%1:25:00::","synset_id":"column.n.04"},{"index":2,"sense_key":"tower%1:06:01::","synset_id":"tugboat.n.01"},{"index":3,"sense_key":"tower%1:06:00::","synset_id":"tower.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"tower%1:06:00::","call_ids":["senseval2.d000.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"tower%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.004654999999999999,"input_uncached_usd":0.002405,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.114690124988556}
{"item_id":"senseval2.d000.s011.t003","gold_sense_keys":["scrape%2:35:03::"],"candidates":[{"index":1,"sense_key":"scrape%2:35:01::","synset_id":"skin.v.02"},{"index":2,"sense_key":"scrape%2:36:00::","synset_id":"scrape.v.02"},{"index":3,"sense_key":"scrape%2:35:00::","synset_id":"scrape.v.01"},{"index":4,"sense_key":"scrape%2:35:03::","synset_id":"scrape.v.05"},{"index":5,"sense_key":"scrape%2:35:02::","synset_id":"scratch.v.02"},{"index":6,"sense_key":"scrape%2:38:00::","synset_id":"scrape.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"scrape%2:35:03::","call_ids":["senseval2.d000.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"scrape%2:35:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":114},"cost":{"currency":"USD","total_usd":0.006895,"input_uncached_usd":0.0029950000000000003,"input_cached_usd":0.0,"output_usd":0.0039000000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.386031166999601}
{"item_id":"senseval2.d000.s011.t004","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":2,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":3,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":4,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":5,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":6,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":7,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":8,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":9,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":10,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":11,"sense_key":"say%2:32:13::","synset_id":"say.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":838,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02003,"input_uncached_usd":0.00419,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.243890749989077}
{"item_id":"senseval2.d000.s011.t006","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"},{"index":2,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"},{"index":3,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"},{"index":4,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"worker%1:18:00::","call_ids":["senseval2.d000.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00432,"input_uncached_usd":0.0026100000000000003,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9955183340935037}
{"item_id":"senseval2.d000.s012.t000","gold_sense_keys":["try%2:41:00::"],"candidates":[{"index":1,"sense_key":"try%2:37:01::","synset_id":"try.v.06"},{"index":2,"sense_key":"try%2:34:00::","synset_id":"sample.v.01"},{"index":3,"sense_key":"try%2:29:00::","synset_id":"try_on.v.01"},{"index":4,"sense_key":"try%2:30:00::","synset_id":"try.v.08"},{"index":5,"sense_key":"try%2:37:00::","synset_id":"try.v.07"},{"index":6,"sense_key":"try%2:41:01::","synset_id":"test.v.01"},{"index":7,"sense_key":"try%2:41:06::","synset_id":"judge.v.05"},{"index":8,"sense_key":"try%2:41:05::","synset_id":"hear.v.03"},{"index":9,"sense_key":"try%2:41:00::","synset_id":"try.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"try%2:41:00::","call_ids":["senseval2.d000.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"try%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":790,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.005390000000000001,"input_uncached_usd":0.00395,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.22402104199864}
{"item_id":"senseval2.d000.s012.t001","gold_sense_keys":["train%2:31:01::","train%2:32:00::","train%2:41:00::"],"candidates":[{"index":1,"sense_key":"train%2:41:00::","synset_id":"prepare.v.05"},{"index":2,"sense_key":"train%2:41:02::","synset_id":"discipline.v.01"},{"index":3,"sense_key":"train%2:31:01::","synset_id":"train.v.01"},{"index":4,"sense_key":"train%2:29:00::","synset_id":"train.v.08"},{"index":5,"sense_key":"train%2:33:00::","synset_id":"aim.v.01"},{"index":6,"sense_key":"train%2:41:03::","synset_id":"train.v.09"},{"index":7,"sense_key":"train%2:32:00::","synset_id":"coach.v.01"},{"index":8,"sense_key":"train%2:35:04::","synset_id":"trail.v.05"},{"index":9,"sense_key":"train%2:31:00::","synset_id":"train.v.02"},{"index":10,"sense_key":"train%2:41:01::","synset_id":"educate.v.03"},{"index":11,"sense_key":"train%2:38:00::","synset_id":"train.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"train%2:31:01::","call_ids":["senseval2.d000.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"train%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":890,"cached_input_tokens":0,"output_tokens":369,"reasoning_output_tokens":354},"cost":{"currency":"USD","total_usd":0.015520000000000003,"input_uncached_usd":0.00445,"input_cached_usd":0.0,"output_usd":0.01107,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.034979791962542}
{"item_id":"senseval2.d000.s012.t003","gold_sense_keys":["disco%1:06:00::"],"candidates":[{"index":1,"sense_key":"disco%1:10:00::","synset_id":"disco.n.01"},{"index":2,"sense_key":"disco%1:06:00::","synset_id":"disco.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"disco%1:06:00::","call_ids":["senseval2.d000.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"disco%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":118},"cost":{"currency":"USD","total_usd":0.00622,"input_uncached_usd":0.0022,"input_cached_usd":0.0,"output_usd":0.00402,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.613239708007313}
{"item_id":"senseval2.d000.s012.t004","gold_sense_keys":["dance%1:11:00::","dance%1:14:00::"],"candidates":[{"index":1,"sense_key":"dance%1:14:00::","synset_id":"dance.n.02"},{"index":2,"sense_key":"dance%1:04:00::","synset_id":"dancing.n.01"},{"index":3,"sense_key":"dance%1:10:00::","synset_id":"dance.n.01"},{"index":4,"sense_key":"dance%1:11:00::","synset_id":"dance.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"dance%1:11:00::","call_ids":["senseval2.d000.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"dance%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":466,"cached_input_tokens":0,"output_tokens":407,"reasoning_output_tokens":391},"cost":{"currency":"USD","total_usd":0.01454,"input_uncached_usd":0.00233,"input_cached_usd":0.0,"output_usd":0.01221,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.619925874983892}
{"item_id":"senseval2.d000.s012.t005","gold_sense_keys":["just%4:02:03::"],"candidates":[{"index":1,"sense_key":"just%4:02:01::","synset_id":"just.r.03"},{"index":2,"sense_key":"just%4:02:06::","synset_id":"just.r.06"},{"index":3,"sense_key":"just%4:02:00::","synset_id":"merely.r.01"},{"index":4,"sense_key":"just%4:02:03::","synset_id":"just.r.04"},{"index":5,"sense_key":"just%4:02:04::","synset_id":"barely.r.01"},{"index":6,"sense_key":"just%4:02:05::","synset_id":"precisely.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"just%4:02:00::","call_ids":["senseval2.d000.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"just%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":209,"reasoning_output_tokens":193},"cost":{"currency":"USD","total_usd":0.009485,"input_uncached_usd":0.0032150000000000004,"input_cached_usd":0.0,"output_usd":0.00627,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.0166760419961065}
{"item_id":"senseval2.d000.s012.t006","gold_sense_keys":["drift%2:38:04::"],"candidates":[{"index":1,"sense_key":"drift%2:42:00::","synset_id":"freewheel.v.01"},{"index":2,"sense_key":"drift%2:30:00::","synset_id":"drift.v.10"},{"index":3,"sense_key":"drift%2:38:04::","synset_id":"stray.v.02"},{"index":4,"sense_key":"drift%2:30:10::","synset_id":"drift.v.09"},{"index":5,"sense_key":"drift%2:38:01::","synset_id":"roll.v.12"},{"index":6,"sense_key":"drift%2:38:06::","synset_id":"drift.v.07"},{"index":7,"sense_key":"drift%2:38:05::","synset_id":"drift.v.06"},{"index":8,"sense_key":"drift%2:35:03::","synset_id":"drift.v.08"},{"index":9,"sense_key":"drift%2:38:02::","synset_id":"float.v.01"},{"index":10,"sense_key":"drift%2:42:03::","synset_id":"drift.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"drift%2:38:04::","call_ids":["senseval2.d000.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"drift%2:38:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":804,"cached_input_tokens":0,"output_tokens":234,"reasoning_output_tokens":219},"cost":{"currency":"USD","total_usd":0.011040000000000001,"input_uncached_usd":0.00402,"input_cached_usd":0.0,"output_usd":0.00702,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.683431665995158}
{"item_id":"senseval2.d000.s012.t007","gold_sense_keys":["away%4:02:00::"],"candidates":[{"index":1,"sense_key":"away%4:02:06::","synset_id":"away.r.10"},{"index":2,"sense_key":"away%4:02:01::","synset_id":"away.r.02"},{"index":3,"sense_key":"away%4:02:10::","synset_id":"away.r.08"},{"index":4,"sense_key":"away%4:02:02::","synset_id":"aside.r.06"},{"index":5,"sense_key":"away%4:02:08::","synset_id":"off.r.02"},{"index":6,"sense_key":"away%4:02:00::","synset_id":"away.r.01"},{"index":7,"sense_key":"away%4:02:05::","synset_id":"away.r.06"},{"index":8,"sense_key":"away%4:02:07::","synset_id":"away.r.09"},{"index":9,"sense_key":"away%4:02:09::","synset_id":"away.r.07"},{"index":10,"sense_key":"away%4:02:03::","synset_id":"aside.r.02"},{"index":11,"sense_key":"away%4:02:04::","synset_id":"away.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"away%4:02:00::","call_ids":["senseval2.d000.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"away%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":838,"cached_input_tokens":0,"output_tokens":282,"reasoning_output_tokens":266},"cost":{"currency":"USD","total_usd":0.012650000000000002,"input_uncached_usd":0.00419,"input_cached_usd":0.0,"output_usd":0.00846,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.406661416986026}
{"item_id":"senseval2.d000.s013.t000","gold_sense_keys":["worry%2:37:00::"],"candidates":[{"index":1,"sense_key":"worry%2:35:00::","synset_id":"worry.v.05"},{"index":2,"sense_key":"worry%2:37:12::","synset_id":"worry.v.02"},{"index":3,"sense_key":"worry%2:37:00::","synset_id":"worry.v.01"},{"index":4,"sense_key":"worry%2:37:01::","synset_id":"worry.v.03"},{"index":5,"sense_key":"worry%2:42:00::","synset_id":"concern.v.02"},{"index":6,"sense_key":"worry%2:35:01::","synset_id":"worry.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"worry%2:37:00::","call_ids":["senseval2.d000.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"worry%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":323,"reasoning_output_tokens":307},"cost":{"currency":"USD","total_usd":0.01241,"input_uncached_usd":0.00272,"input_cached_usd":0.0,"output_usd":0.00969,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.574467375059612}
{"item_id":"senseval2.d000.s013.t003","gold_sense_keys":["youth%1:07:00::","youth%1:14:00::","youth%1:26:00::","youth%1:28:00::"],"candidates":[{"index":1,"sense_key":"youth%1:26:00::","synset_id":"youth.n.04"},{"index":2,"sense_key":"youth%1:28:00::","synset_id":"youth.n.03"},{"index":3,"sense_key":"youth%1:28:01::","synset_id":"youth.n.05"},{"index":4,"sense_key":"youth%1:07:00::","synset_id":"youth.n.06"},{"index":5,"sense_key":"youth%1:18:00::","synset_id":"young_person.n.01"},{"index":6,"sense_key":"youth%1:14:00::","synset_id":"young.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"youth%1:26:00::","call_ids":["senseval2.d000.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"youth%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":441,"reasoning_output_tokens":426},"cost":{"currency":"USD","total_usd":0.01586,"input_uncached_usd":0.0026300000000000004,"input_cached_usd":0.0,"output_usd":0.01323,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.274872333975509}
{"item_id":"senseval2.d000.s013.t005","gold_sense_keys":["rank%1:14:02::"],"candidates":[{"index":1,"sense_key":"rank%1:14:02::","synset_id":"membership.n.01"},{"index":2,"sense_key":"rank%1:14:01::","synset_id":"rank_and_file.n.01"},{"index":3,"sense_key":"rank%1:14:00::","synset_id":"rank.n.01"},{"index":4,"sense_key":"rank%1:26:00::","synset_id":"rank.n.02"},{"index":5,"sense_key":"rank%1:26:01::","synset_id":"social_station.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rank%1:14:02::","call_ids":["senseval2.d000.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rank%1:14:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.00544,"input_uncached_usd":0.0028000000000000004,"input_cached_usd":0.0,"output_usd":0.00264,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3289405420655385}
{"item_id":"senseval2.d000.s013.t006","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["senseval2.d000.s013.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00401,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.083554750075564}
{"item_id":"senseval2.d000.s013.t007","gold_sense_keys":["keep%2:42:00::"],"candidates":[{"index":1,"sense_key":"keep%2:35:10::","synset_id":"keep.v.21"},{"index":2,"sense_key":"keep%2:32:00::","synset_id":"keep.v.08"},{"index":3,"sense_key":"keep%2:42:03::","synset_id":"keep.v.12"},{"index":4,"sense_key":"keep%2:41:05::","synset_id":"observe.v.06"},{"index":5,"sense_key":"keep%2:42:12::","synset_id":"keep.v.16"},{"index":6,"sense_key":"keep%2:42:02::","synset_id":"keep.v.15"},{"index":7,"sense_key":"keep%2:30:10::","synset_id":"preserve.v.04"},{"index":8,"sense_key":"keep%2:42:00::","synset_id":"keep.v.01"},{"index":9,"sense_key":"keep%2:40:10::","synset_id":"keep.v.20"},{"index":10,"sense_key":"keep%2:40:01::","synset_id":"keep.v.07"},{"index":11,"sense_key":"keep%2:31:00::","synset_id":"observe.v.08"},{"index":12,"sense_key":"keep%2:40:00::","synset_id":"keep.v.03"},{"index":13,"sense_key":"keep%2:40:12::","synset_id":"keep.v.19"},{"index":14,"sense_key":"keep%2:41:02::","synset_id":"retain.v.02"},{"index":15,"sense_key":"keep%2:34:00::","synset_id":"sustain.v.04"},{"index":16,"sense_key":"keep%2:41:00::","synset_id":"restrain.v.01"},{"index":17,"sense_key":"keep%2:42:01::","synset_id":"keep.v.09"},{"index":18,"sense_key":"keep%2:42:07::","synset_id":"continue.v.01"},{"index":19,"sense_key":"keep%2:41:03::","synset_id":"observe.v.09"},{"index":20,"sense_key":"keep%2:40:09::","synset_id":"keep_open.v.01"},{"index":21,"sense_key":"keep%2:40:13::","synset_id":"keep.v.18"},{"index":22,"sense_key":"keep%2:41:01::","synset_id":"prevent.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"keep%2:42:00::","call_ids":["senseval2.d000.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"keep%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1256,"cached_input_tokens":0,"output_tokens":250,"reasoning_output_tokens":234},"cost":{"currency":"USD","total_usd":0.01378,"input_uncached_usd":0.006280000000000001,"input_cached_usd":0.0,"output_usd":0.007500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.7515451250365}
{"item_id":"senseval2.d000.s013.t008","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":2,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":3,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":4,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":5,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":6,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":7,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":8,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":9,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"},{"index":10,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s013.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":752,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":138},"cost":{"currency":"USD","total_usd":0.008380000000000002,"input_uncached_usd":0.0037600000000000003,"input_cached_usd":0.0,"output_usd":0.00462,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.562023082980886}
{"item_id":"senseval2.d000.s014.t000","gold_sense_keys":["history%1:28:00::"],"candidates":[{"index":1,"sense_key":"history%1:28:02::","synset_id":"history.n.04"},{"index":2,"sense_key":"history%1:09:00::","synset_id":"history.n.03"},{"index":3,"sense_key":"history%1:28:00::","synset_id":"history.n.01"},{"index":4,"sense_key":"history%1:10:00::","synset_id":"history.n.02"},{"index":5,"sense_key":"history%1:09:01::","synset_id":"history.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"history%1:28:02::","call_ids":["senseval2.d000.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"history%1:28:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.034155,"input_uncached_usd":0.002775,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":19.89679645805154}
{"item_id":"senseval2.d000.s014.t001","gold_sense_keys":["side%1:10:00::","side%1:14:00::"],"candidates":[{"index":1,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":2,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":3,"sense_key":"side%1:11:00::","synset_id":"english.n.04"},{"index":4,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":5,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":6,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":7,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":8,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":9,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":10,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":11,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":12,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"side%1:14:00::","call_ids":["senseval2.d000.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"side%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":849,"cached_input_tokens":0,"output_tokens":504,"reasoning_output_tokens":488},"cost":{"currency":"USD","total_usd":0.019365,"input_uncached_usd":0.0042450000000000005,"input_cached_usd":0.0,"output_usd":0.01512,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.764802750083618}
{"item_id":"senseval2.d000.s015.t001","gold_sense_keys":["survey%1:04:02::"],"candidates":[{"index":1,"sense_key":"survey%1:04:01::","synset_id":"view.n.03"},{"index":2,"sense_key":"survey%1:10:00::","synset_id":"sketch.n.03"},{"index":3,"sense_key":"survey%1:04:02::","synset_id":"survey.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"survey%1:04:02::","call_ids":["senseval2.d000.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"survey%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":112},"cost":{"currency":"USD","total_usd":0.006105,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.00384,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.94391233404167}
{"item_id":"senseval2.d000.s015.t002","gold_sense_keys":["take%2:41:04::"],"candidates":[{"index":1,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":2,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":3,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":4,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":5,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":6,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":7,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":8,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":9,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":10,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"},{"index":11,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":12,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":13,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":14,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":15,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":16,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":17,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":18,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":19,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":20,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":21,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":22,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":23,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":24,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":25,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":26,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":27,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":28,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":29,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":30,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":31,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":32,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":33,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":34,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":35,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":36,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":37,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":38,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":39,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":40,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":41,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":42,"sense_key":"take%2:30:09::","synset_id":"take.v.29"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":35,"chosen_sense_key":"take%2:41:04::","call_ids":["senseval2.d000.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":35,"predicted_sense_key":"take%2:41:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2211,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.026865,"input_uncached_usd":0.011055,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.17707158299163}
{"item_id":"senseval2.d000.s015.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":2,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":3,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d000.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.0053,"input_uncached_usd":0.00272,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5465879160910845}
{"item_id":"senseval2.d000.s015.t007","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":2,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":3,"sense_key":"church%1:14:01::","synset_id":"church.n.04"},{"index":4,"sense_key":"church%1:06:00::","synset_id":"church.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.00544,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.032399540999904}
{"item_id":"senseval2.d000.s015.t008","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":4,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":7,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":8,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":9,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":10,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s015.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":776,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.00604,"input_uncached_usd":0.00388,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3150632079923525}
{"item_id":"senseval2.d000.s015.t009","gold_sense_keys":["no%4:02:01::"],"candidates":[{"index":1,"sense_key":"no%4:02:02::","synset_id":"no.r.01"},{"index":2,"sense_key":"no%4:02:01::","synset_id":"no.r.02"},{"index":3,"sense_key":"no%4:02:00::","synset_id":"no.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"no%4:02:02::","call_ids":["senseval2.d000.s015.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"no%4:02:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":463,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018155,"input_uncached_usd":0.0023150000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.230409041047096}
{"item_id":"senseval2.d000.s015.t011","gold_sense_keys":["ring%2:39:01::"],"candidates":[{"index":1,"sense_key":"ring%2:39:04::","synset_id":"resound.v.01"},{"index":2,"sense_key":"ring%2:39:00::","synset_id":"ring.v.01"},{"index":3,"sense_key":"ring%2:35:03::","synset_id":"surround.v.01"},{"index":4,"sense_key":"ring%2:32:00::","synset_id":"call.v.03"},{"index":5,"sense_key":"ring%2:35:00::","synset_id":"ring.v.06"},{"index":6,"sense_key":"ring%2:39:01::","synset_id":"ring.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"ring%2:39:01::","call_ids":["senseval2.d000.s015.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"ring%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.0061200000000000004,"input_uncached_usd":0.0031500000000000005,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.771697083022445}
{"item_id":"senseval2.d000.s015.t012","gold_sense_keys":["sunday%1:28:00::"],"candidates":[{"index":1,"sense_key":"sunday%1:18:00::","synset_id":"sunday.n.02"},{"index":2,"sense_key":"sunday%1:28:00::","synset_id":"sunday.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sunday%1:28:00::","call_ids":["senseval2.d000.s015.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sunday%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":431,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.0038050000000000002,"input_uncached_usd":0.0021550000000000002,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.592461958061904}
{"item_id":"senseval2.d000.s015.t013","gold_sense_keys":["ring%2:39:01::"],"candidates":[{"index":1,"sense_key":"ring%2:32:00::","synset_id":"call.v.03"},{"index":2,"sense_key":"ring%2:39:04::","synset_id":"resound.v.01"},{"index":3,"sense_key":"ring%2:39:01::","synset_id":"ring.v.03"},{"index":4,"sense_key":"ring%2:39:00::","synset_id":"ring.v.01"},{"index":5,"sense_key":"ring%2:35:03::","synset_id":"surround.v.01"},{"index":6,"sense_key":"ring%2:35:00::","synset_id":"ring.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ring%2:39:01::","call_ids":["senseval2.d000.s015.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ring%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.005695,"input_uncached_usd":0.0031450000000000002,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.660287667065859}
{"item_id":"senseval2.d000.s016.t000","gold_sense_keys":["easy%3:00:01::"],"candidates":[{"index":1,"sense_key":"easy%3:00:02::","synset_id":"easy.a.03"},{"index":2,"sense_key":"easy%5:00:00:abundant:00","synset_id":"easy.s.10"},{"index":3,"sense_key":"easy%5:00:00:gradual:02","synset_id":"easy.s.06"},{"index":4,"sense_key":"easy%5:00:00:light:08","synset_id":"easy.s.03"},{"index":5,"sense_key":"easy%5:00:00:comfortable:00","synset_id":"easy.s.07"},{"index":6,"sense_key":"easy%5:00:00:unhurried:00","synset_id":"easy.s.01"},{"index":7,"sense_key":"easy%5:00:00:available:00","synset_id":"easy.s.09"},{"index":8,"sense_key":"easy%5:00:00:pleasing:00","synset_id":"easy.s.02"},{"index":9,"sense_key":"easy%5:00:00:unchaste:00","synset_id":"easy.s.08"},{"index":10,"sense_key":"easy%5:00:00:impressionable:00","synset_id":"easy.s.04"},{"index":11,"sense_key":"easy%3:00:01::","synset_id":"easy.a.01"},{"index":12,"sense_key":"easy%5:00:00:rich:00","synset_id":"comfortable.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"easy%3:00:01::","call_ids":["senseval2.d000.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"easy%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":845,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.005815000000000001,"input_uncached_usd":0.0042250000000000005,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9871810829499736}
{"item_id":"senseval2.d000.s016.t001","gold_sense_keys":["see%2:31:01::"],"candidates":[{"index":1,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":4,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":5,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":6,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":7,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":8,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":9,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":10,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"},{"index":11,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":12,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":13,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":14,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":15,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":16,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":17,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":18,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":19,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":20,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":21,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":22,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":23,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":24,"sense_key":"see%2:39:11::","synset_id":"see.v.19"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"see%2:31:01::","call_ids":["senseval2.d000.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"see%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1472,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.009940000000000001,"input_uncached_usd":0.00736,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3633443330181763}
{"item_id":"senseval2.d000.s016.t002","gold_sense_keys":["ancient%5:00:00:old:02"],"candidates":[{"index":1,"sense_key":"ancient%5:00:00:past:00","synset_id":"ancient.s.01"},{"index":2,"sense_key":"ancient%5:00:00:old:02","synset_id":"ancient.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ancient%5:00:00:old:02","call_ids":["senseval2.d000.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ancient%5:00:00:old:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":598,"reasoning_output_tokens":575},"cost":{"currency":"USD","total_usd":0.020050000000000002,"input_uncached_usd":0.0021100000000000003,"input_cached_usd":0.0,"output_usd":0.01794,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.518274792004377}
{"item_id":"senseval2.d000.s016.t003","gold_sense_keys":["art%1:09:00::"],"candidates":[{"index":1,"sense_key":"art%1:10:00::","synset_id":"artwork.n.01"},{"index":2,"sense_key":"art%1:06:00::","synset_id":"art.n.01"},{"index":3,"sense_key":"art%1:09:00::","synset_id":"art.n.03"},{"index":4,"sense_key":"art%1:04:00::","synset_id":"art.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"art%1:09:00::","call_ids":["senseval2.d000.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"art%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.005065,"input_uncached_usd":0.002635,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5517483750591055}
{"item_id":"senseval2.d000.s017.t000","gold_sense_keys":["less%4:02:00::"],"candidates":[{"index":1,"sense_key":"less%4:02:01::","synset_id":"less.r.02"},{"index":2,"sense_key":"less%4:02:00::","synset_id":"less.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"less%4:02:00::","call_ids":["senseval2.d000.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"less%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":413,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":81},"cost":{"currency":"USD","total_usd":0.004975,"input_uncached_usd":0.002065,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9822722079697996}
{"item_id":"senseval2.d000.s017.t002","gold_sense_keys":["version%1:09:01::"],"candidates":[{"index":1,"sense_key":"version%1:10:02::","synset_id":"adaptation.n.01"},{"index":2,"sense_key":"version%1:10:01::","synset_id":"version.n.01"},{"index":3,"sense_key":"version%1:09:01::","synset_id":"version.n.02"},{"index":4,"sense_key":"version%1:04:00::","synset_id":"version.n.06"},{"index":5,"sense_key":"version%1:10:00::","synset_id":"translation.n.01"},{"index":6,"sense_key":"version%1:09:00::","synset_id":"interpretation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"version%1:09:01::","call_ids":["senseval2.d000.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"version%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.005825,"input_uncached_usd":0.002915,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5463624160038307}
{"item_id":"senseval2.d000.s017.t003","gold_sense_keys":["play%2:36:00::","play%2:36:01::","play%2:36:12::"],"candidates":[{"index":1,"sense_key":"play%2:33:03::","synset_id":"play.v.15"},{"index":2,"sense_key":"play%2:41:13::","synset_id":"dally.v.01"},{"index":3,"sense_key":"play%2:36:05::","synset_id":"play.v.18"},{"index":4,"sense_key":"play%2:33:13::","synset_id":"play.v.32"},{"index":5,"sense_key":"play%2:41:03::","synset_id":"play.v.05"},{"index":6,"sense_key":"play%2:33:02::","synset_id":"meet.v.10"},{"index":7,"sense_key":"play%2:29:04::","synset_id":"play.v.35"},{"index":8,"sense_key":"play%2:33:07::","synset_id":"play.v.29"},{"index":9,"sense_key":"play%2:36:12::","synset_id":"play.v.03"},{"index":10,"sense_key":"play%2:41:12::","synset_id":"play.v.02"},{"index":11,"sense_key":"play%2:41:02::","synset_id":"play.v.17"},{"index":12,"sense_key":"play%2:33:09::","synset_id":"play.v.31"},{"index":13,"sense_key":"play%2:36:13::","synset_id":"bring.v.03"},{"index":14,"sense_key":"play%2:35:07::","synset_id":"play.v.28"},{"index":15,"sense_key":"play%2:35:00::","synset_id":"toy.v.02"},{"index":16,"sense_key":"play%2:33:00::","synset_id":"play.v.01"},{"index":17,"sense_key":"play%2:36:03::","synset_id":"play.v.14"},{"index":18,"sense_key":"play%2:36:04::","synset_id":"play.v.26"},{"index":19,"sense_key":"play%2:38:03::","synset_id":"play.v.24"},{"index":20,"sense_key":"play%2:31:01::","synset_id":"dally.v.04"},{"index":21,"sense_key":"play%2:33:01::","synset_id":"play.v.10"},{"index":22,"sense_key":"play%2:33:12::","synset_id":"play.v.33"},{"index":23,"sense_key":"play%2:36:00::","synset_id":"play.v.06"},{"index":24,"sense_key":"play%2:42:00::","synset_id":"play.v.22"},{"index":25,"sense_key":"play%2:36:06::","synset_id":"play.v.12"},{"index":26,"sense_key":"play%2:36:10::","synset_id":"act.v.10"},{"index":27,"sense_key":"play%2:29:00::","synset_id":"play.v.16"},{"index":28,"sense_key":"play%2:36:02::","synset_id":"act.v.03"},{"index":29,"sense_key":"play%2:34:13::","synset_id":"play.v.20"},{"index":30,"sense_key":"play%2:29:01::","synset_id":"act.v.05"},{"index":31,"sense_key":"play%2:36:11::","synset_id":"play.v.13"},{"index":32,"sense_key":"play%2:41:00::","synset_id":"play.v.11"},{"index":33,"sense_key":"play%2:38:00::","synset_id":"play.v.09"},{"index":34,"sense_key":"play%2:36:01::","synset_id":"play.v.07"},{"index":35,"sense_key":"play%2:33:08::","synset_id":"bet.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":23,"chosen_sense_key":"play%2:36:00::","call_ids":["senseval2.d000.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":23,"predicted_sense_key":"play%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1871,"cached_input_tokens":0,"output_tokens":1007,"reasoning_output_tokens":984},"cost":{"currency":"USD","total_usd":0.039565,"input_uncached_usd":0.009355,"input_cached_usd":0.0,"output_usd":0.03021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":19.361725916038267}
{"item_id":"senseval2.d000.s017.t004","gold_sense_keys":["tune%1:10:00::"],"candidates":[{"index":1,"sense_key":"tune%1:07:00::","synset_id":"tune.n.02"},{"index":2,"sense_key":"tune%1:10:00::","synset_id":"tune.n.01"},{"index":3,"sense_key":"tune%1:04:00::","synset_id":"tune.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tune%1:10:00::","call_ids":["senseval2.d000.s017.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tune%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.003980000000000001,"input_uncached_usd":0.0023000000000000004,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.424999332986772}
{"item_id":"senseval2.d000.s017.t005","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":2,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":3,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":4,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"},{"index":5,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":6,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":7,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019645,"input_uncached_usd":0.0038050000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.958176541957073}
{"item_id":"senseval2.d000.s017.t008","gold_sense_keys":["europe%1:14:01::","europe%1:17:00::"],"candidates":[{"index":1,"sense_key":"europe%1:14:01::","synset_id":"europe.n.03"},{"index":2,"sense_key":"europe%1:17:00::","synset_id":"europe.n.01"},{"index":3,"sense_key":"europe%1:14:00::","synset_id":"european_union.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"europe%1:17:00::","call_ids":["senseval2.d000.s017.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"europe%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.004545,"input_uncached_usd":0.002445,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5299456670181826}
{"item_id":"senseval2.d000.s017.t009","gold_sense_keys":["consider%2:31:00::"],"candidates":[{"index":1,"sense_key":"consider%2:31:08::","synset_id":"think.v.01"},{"index":2,"sense_key":"consider%2:32:02::","synset_id":"consider.v.05"},{"index":3,"sense_key":"consider%2:39:00::","synset_id":"study.v.03"},{"index":4,"sense_key":"consider%2:31:05::","synset_id":"consider.v.09"},{"index":5,"sense_key":"consider%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"consider%2:39:04::","synset_id":"regard.v.02"},{"index":7,"sense_key":"consider%2:31:01::","synset_id":"consider.v.03"},{"index":8,"sense_key":"consider%2:39:08::","synset_id":"view.v.02"},{"index":9,"sense_key":"consider%2:32:00::","synset_id":"consider.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"consider%2:31:00::","call_ids":["senseval2.d000.s017.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"consider%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":708,"cached_input_tokens":0,"output_tokens":1286,"reasoning_output_tokens":1258},"cost":{"currency":"USD","total_usd":0.042120000000000005,"input_uncached_usd":0.00354,"input_cached_usd":0.0,"output_usd":0.03858,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":23.519234457984567}
{"item_id":"senseval2.d000.s017.t010","gold_sense_keys":["english%1:18:00::"],"candidates":[{"index":1,"sense_key":"english%1:18:00::","synset_id":"english.n.02"},{"index":2,"sense_key":"english%1:11:00::","synset_id":"english.n.04"},{"index":3,"sense_key":"english%1:09:00::","synset_id":"english.n.03"},{"index":4,"sense_key":"english%1:10:00::","synset_id":"english.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"english%1:18:00::","call_ids":["senseval2.d000.s017.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"english%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.00383,"input_uncached_usd":0.00239,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.87321216694545}
{"item_id":"senseval2.d000.s017.t012","gold_sense_keys":["fit%3:00:02::"],"candidates":[{"index":1,"sense_key":"fit%3:00:02::","synset_id":"fit.a.01"},{"index":2,"sense_key":"fit%3:00:01::","synset_id":"fit.a.03"},{"index":3,"sense_key":"fit%5:00:00:ready:00","synset_id":"fit.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fit%3:00:02::","call_ids":["senseval2.d000.s017.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fit%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.0043950000000000005,"input_uncached_usd":0.0023550000000000003,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.227669500047341}
{"item_id":"senseval2.d000.s017.t013","gold_sense_keys":["only%4:02:01::"],"candidates":[{"index":1,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":2,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":3,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:05::","synset_id":"only.r.07"},{"index":6,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":7,"sense_key":"only%4:02:03::","synset_id":"only.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"only%4:02:01::","call_ids":["senseval2.d000.s017.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"only%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.005350000000000001,"input_uncached_usd":0.0030700000000000002,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6967783749569207}
{"item_id":"senseval2.d000.s017.t014","gold_sense_keys":["foreigner%1:18:00::"],"candidates":[{"index":1,"sense_key":"foreigner%1:18:00::","synset_id":"foreigner.n.01"},{"index":2,"sense_key":"foreigner%1:18:01::","synset_id":"foreigner.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"foreigner%1:18:00::","call_ids":["senseval2.d000.s017.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"foreigner%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":94},"cost":{"currency":"USD","total_usd":0.005355,"input_uncached_usd":0.002085,"input_cached_usd":0.0,"output_usd":0.00327,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.6711482079699636}
{"item_id":"senseval2.d000.s018.t002","gold_sense_keys":["english%1:18:00::"],"candidates":[{"index":1,"sense_key":"english%1:09:00::","synset_id":"english.n.03"},{"index":2,"sense_key":"english%1:11:00::","synset_id":"english.n.04"},{"index":3,"sense_key":"english%1:10:00::","synset_id":"english.n.01"},{"index":4,"sense_key":"english%1:18:00::","synset_id":"english.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"english%1:18:00::","call_ids":["senseval2.d000.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"english%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.0037900000000000004,"input_uncached_usd":0.0024700000000000004,"input_cached_usd":0.0,"output_usd":0.00132,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.077900333912112}
{"item_id":"senseval2.d000.s018.t003","gold_sense_keys":["invent%2:36:00::"],"candidates":[{"index":1,"sense_key":"invent%2:36:01::","synset_id":"fabricate.v.02"},{"index":2,"sense_key":"invent%2:36:00::","synset_id":"invent.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"invent%2:36:00::","call_ids":["senseval2.d000.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"invent%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.0044,"input_uncached_usd":0.0022700000000000003,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.292094041942619}
{"item_id":"senseval2.d000.s018.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d000.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.004620000000000001,"input_uncached_usd":0.0027300000000000002,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.088726625079289}
{"item_id":"senseval2.d000.s018.t006","gold_sense_keys":["require%2:34:00::","require%2:42:00::"],"candidates":[{"index":1,"sense_key":"require%2:32:00::","synset_id":"command.v.02"},{"index":2,"sense_key":"require%2:32:01::","synset_id":"ask.v.04"},{"index":3,"sense_key":"require%2:42:00::","synset_id":"necessitate.v.01"},{"index":4,"sense_key":"require%2:34:00::","synset_id":"want.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"require%2:42:00::","call_ids":["senseval2.d000.s018.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"require%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.00477,"input_uncached_usd":0.0025800000000000003,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3795131670776755}
{"item_id":"senseval2.d000.s018.t007","gold_sense_keys":["physical%3:00:00::"],"candidates":[{"index":1,"sense_key":"physical%5:00:02:material:01","synset_id":"physical.s.05"},{"index":2,"sense_key":"physical%5:00:00:energetic:00","synset_id":"physical.s.03"},{"index":3,"sense_key":"physical%5:00:00:material:01","synset_id":"physical.s.01"},{"index":4,"sense_key":"physical%3:01:00::","synset_id":"physical.a.02"},{"index":5,"sense_key":"physical%5:00:00:forceful:00","synset_id":"forcible.s.01"},{"index":6,"sense_key":"physical%5:00:00:natural:03","synset_id":"physical.s.02"},{"index":7,"sense_key":"physical%3:00:00::","synset_id":"physical.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"physical%3:00:00::","call_ids":["senseval2.d000.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"physical%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.0063750000000000005,"input_uncached_usd":0.0031650000000000003,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.029245416983031}
{"item_id":"senseval2.d000.s018.t009","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":2,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":3,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":4,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":5,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":6,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"},{"index":7,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":8,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s018.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":778,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.0061400000000000005,"input_uncached_usd":0.0038900000000000002,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4871240420034155}
{"item_id":"senseval2.d000.s018.t010","gold_sense_keys":["weigh%2:42:01::"],"candidates":[{"index":1,"sense_key":"weigh%2:42:04::","synset_id":"weigh.v.05"},{"index":2,"sense_key":"weigh%2:32:00::","synset_id":"consider.v.04"},{"index":3,"sense_key":"weigh%2:42:03::","synset_id":"count.v.02"},{"index":4,"sense_key":"weigh%2:42:01::","synset_id":"weigh.v.01"},{"index":5,"sense_key":"weigh%2:42:00::","synset_id":"weigh.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"weigh%2:42:01::","call_ids":["senseval2.d000.s018.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"weigh%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.004155000000000001,"input_uncached_usd":0.0027150000000000004,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1533178329700604}
{"item_id":"senseval2.d000.s018.t014","gold_sense_keys":["intense%3:00:00::"],"candidates":[{"index":1,"sense_key":"intense%5:00:00:saturated:03","synset_id":"intense.s.02"},{"index":2,"sense_key":"intense%5:00:00:sharp:04","synset_id":"acute.s.01"},{"index":3,"sense_key":"intense%3:00:00::","synset_id":"intense.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"intense%3:00:00::","call_ids":["senseval2.d000.s018.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"intense%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.00382,"input_uncached_usd":0.00232,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9108005419839174}
{"item_id":"senseval2.d000.s018.t015","gold_sense_keys":["mental%3:00:00::","mental%3:01:00::"],"candidates":[{"index":1,"sense_key":"mental%3:00:00::","synset_id":"mental.a.01"},{"index":2,"sense_key":"mental%3:01:03::","synset_id":"mental.a.03"},{"index":3,"sense_key":"mental%5:00:00:unhealthy:00","synset_id":"mental.s.01"},{"index":4,"sense_key":"mental%3:01:04::","synset_id":"genial.a.02"},{"index":5,"sense_key":"mental%3:01:00::","synset_id":"mental.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mental%3:00:00::","call_ids":["senseval2.d000.s018.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mental%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":523,"cached_input_tokens":0,"output_tokens":141,"reasoning_output_tokens":125},"cost":{"currency":"USD","total_usd":0.006845,"input_uncached_usd":0.002615,"input_cached_usd":0.0,"output_usd":0.00423,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.142789167002775}
{"item_id":"senseval2.d000.s018.t016","gold_sense_keys":["concentration%1:07:01::","concentration%1:09:00::"],"candidates":[{"index":1,"sense_key":"concentration%1:09:00::","synset_id":"concentration.n.05"},{"index":2,"sense_key":"concentration%1:04:02::","synset_id":"concentration.n.06"},{"index":3,"sense_key":"concentration%1:07:03::","synset_id":"concentration.n.02"},{"index":4,"sense_key":"concentration%1:07:01::","synset_id":"assiduity.n.01"},{"index":5,"sense_key":"concentration%1:04:01::","synset_id":"concentration.n.03"},{"index":6,"sense_key":"concentration%1:07:02::","synset_id":"concentration.n.01"},{"index":7,"sense_key":"concentration%1:11:00::","synset_id":"concentration.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"concentration%1:09:00::","call_ids":["senseval2.d000.s018.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"concentration%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.0052,"input_uncached_usd":0.00286,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3119865000480786}
{"item_id":"senseval2.d000.s019.t000","gold_sense_keys":["english%3:01:00::"],"candidates":[{"index":1,"sense_key":"english%3:01:01::","synset_id":"english.a.02"},{"index":2,"sense_key":"english%3:01:00::","synset_id":"english.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"english%3:01:00::","call_ids":["senseval2.d000.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"english%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.004465,"input_uncached_usd":0.0020350000000000004,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2884953329339623}
{"item_id":"senseval2.d000.s019.t001","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":2,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":3,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"},{"index":4,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":5,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":6,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":7,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":8,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":9,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":10,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":98},"cost":{"currency":"USD","total_usd":0.00727,"input_uncached_usd":0.00385,"input_cached_usd":0.0,"output_usd":0.0034200000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.295169875025749}
{"item_id":"senseval2.d000.s019.t003","gold_sense_keys":["round%1:28:01::"],"candidates":[{"index":1,"sense_key":"round%1:28:01::","synset_id":"cycle.n.01"},{"index":2,"sense_key":"round%1:04:03::","synset_id":"round_of_golf.n.01"},{"index":3,"sense_key":"round%1:04:01::","synset_id":"round.n.06"},{"index":4,"sense_key":"round%1:17:00::","synset_id":"round.n.08"},{"index":5,"sense_key":"round%1:06:03::","synset_id":"rung.n.01"},{"index":6,"sense_key":"round%1:04:00::","synset_id":"round.n.04"},{"index":7,"sense_key":"round%1:28:00::","synset_id":"turn.n.09"},{"index":8,"sense_key":"round%1:13:01::","synset_id":"round.n.09"},{"index":9,"sense_key":"round%1:06:00::","synset_id":"circle.n.08"},{"index":10,"sense_key":"round%1:10:01::","synset_id":"round.n.12"},{"index":11,"sense_key":"round%1:15:00::","synset_id":"beat.n.01"},{"index":12,"sense_key":"round%1:13:00::","synset_id":"round.n.10"},{"index":13,"sense_key":"round%1:06:01::","synset_id":"round.n.01"},{"index":14,"sense_key":"round%1:10:00::","synset_id":"round.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"round%1:28:01::","call_ids":["senseval2.d000.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"round%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":886,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02027,"input_uncached_usd":0.004430000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.561089957947843}
{"item_id":"senseval2.d000.s019.t004","gold_sense_keys":["high-pitched%3:00:00::"],"candidates":[{"index":1,"sense_key":"high-pitched%3:00:00::","synset_id":"high.a.04"},{"index":2,"sense_key":"high-pitched%5:00:00:inclined:01","synset_id":"high-pitched.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"high-pitched%3:00:00::","call_ids":["senseval2.d000.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"high-pitched%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.0038350000000000003,"input_uncached_usd":0.002095,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9978471249341965}
{"item_id":"senseval2.d000.s019.t005","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":2,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":3,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":4,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"},{"index":5,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":6,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":7,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":8,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":9,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":10,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.00655,"input_uncached_usd":0.00385,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.706693542073481}
{"item_id":"senseval2.d000.s019.t006","gold_sense_keys":["low%3:00:03::"],"candidates":[{"index":1,"sense_key":"low%5:00:00:dejected:00","synset_id":"gloomy.s.02"},{"index":2,"sense_key":"low%3:00:02::","synset_id":"low.a.01"},{"index":3,"sense_key":"low%5:00:00:contemptible:00","synset_id":"abject.s.01"},{"index":4,"sense_key":"low%3:00:01::","synset_id":"low.a.02"},{"index":5,"sense_key":"low%5:00:00:insufficient:00","synset_id":"depleted.s.01"},{"index":6,"sense_key":"low%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":7,"sense_key":"low%5:00:00:unrefined:01","synset_id":"low.s.02"},{"index":8,"sense_key":"low%5:00:00:soft:04","synset_id":"low.s.01"},{"index":9,"sense_key":"low%3:00:03::","synset_id":"low.a.05"},{"index":10,"sense_key":"low%5:00:00:humble:00","synset_id":"broken.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"low%3:00:03::","call_ids":["senseval2.d000.s019.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"low%3:00:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.006555,"input_uncached_usd":0.003945000000000001,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.009258250007406}
{"item_id":"senseval2.d000.s019.t007","gold_sense_keys":["simple%3:00:02::","simple%5:00:00:easy:01"],"candidates":[{"index":1,"sense_key":"simple%5:00:00:plain:01","synset_id":"simple.s.05"},{"index":2,"sense_key":"simple%3:00:01::","synset_id":"simple.a.06"},{"index":3,"sense_key":"simple%5:00:00:naive:00","synset_id":"childlike.s.02"},{"index":4,"sense_key":"simple%5:00:00:easy:01","synset_id":"elementary.s.01"},{"index":5,"sense_key":"simple%5:00:00:retarded:00","synset_id":"dim-witted.s.01"},{"index":6,"sense_key":"simple%5:00:02:plain:01","synset_id":"bare.s.04"},{"index":7,"sense_key":"simple%3:00:02::","synset_id":"simple.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"simple%3:00:02::","call_ids":["senseval2.d000.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"simple%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019075,"input_uncached_usd":0.0032350000000000005,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.570029124966823}
{"item_id":"senseval2.d000.s019.t009","gold_sense_keys":["scale%1:10:00::"],"candidates":[{"index":1,"sense_key":"scale%1:06:02::","synset_id":"scale.n.07"},{"index":2,"sense_key":"scale%1:06:03::","synset_id":"plate.n.14"},{"index":3,"sense_key":"scale%1:24:00::","synset_id":"scale.n.02"},{"index":4,"sense_key":"scale%1:17:00::","synset_id":"scale.n.05"},{"index":5,"sense_key":"scale%1:10:00::","synset_id":"scale.n.06"},{"index":6,"sense_key":"scale%1:20:00::","synset_id":"scale.n.04"},{"index":7,"sense_key":"scale%1:06:00::","synset_id":"scale.n.08"},{"index":8,"sense_key":"scale%1:05:00::","synset_id":"scale.n.10"},{"index":9,"sense_key":"scale%1:24:03::","synset_id":"scale.n.01"},{"index":10,"sense_key":"scale%1:24:01::","synset_id":"scale.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"scale%1:10:00::","call_ids":["senseval2.d000.s019.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"scale%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.005305000000000001,"input_uncached_usd":0.0035350000000000004,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.267478625057265}
{"item_id":"senseval2.d000.s019.t010","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":4,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:34:00::","synset_id":"use.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d000.s019.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.005875,"input_uncached_usd":0.003115,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.366391250048764}
{"item_id":"senseval2.d000.s019.t012","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":2,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":3,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s019.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00442,"input_uncached_usd":0.0024700000000000004,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.254068832960911}
{"item_id":"senseval2.d000.s019.t014","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":2,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":3,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":4,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":5,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":6,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":7,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":8,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":9,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s019.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":769,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.005705,"input_uncached_usd":0.0038450000000000003,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4062516660196707}
{"item_id":"senseval2.d000.s020.t000","gold_sense_keys":["then%4:02:00::"],"candidates":[{"index":1,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":2,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":3,"sense_key":"then%4:02:01::","synset_id":"then.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"then%4:02:00::","call_ids":["senseval2.d000.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"then%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.00477,"input_uncached_usd":0.00237,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.479744124924764}
{"item_id":"senseval2.d000.s020.t001","gold_sense_keys":["signal%1:10:00::","signal%1:16:00::"],"candidates":[{"index":1,"sense_key":"signal%1:16:00::","synset_id":"signal.n.02"},{"index":2,"sense_key":"signal%1:19:00::","synset_id":"signal.n.03"},{"index":3,"sense_key":"signal%1:10:00::","synset_id":"signal.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"signal%1:16:00::","call_ids":["senseval2.d000.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"signal%1:16:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":104},"cost":{"currency":"USD","total_usd":0.00596,"input_uncached_usd":0.00236,"input_cached_usd":0.0,"output_usd":0.0036,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.984675957937725}
{"item_id":"senseval2.d000.s020.t002","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":2,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":3,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"},{"index":4,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.004370000000000001,"input_uncached_usd":0.0024200000000000003,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1368440000806004}
{"item_id":"senseval2.d000.s020.t003","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":2,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":4,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":7,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":8,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"},{"index":9,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":10,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"begin%2:30:00::","call_ids":["senseval2.d000.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"begin%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":854,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":98},"cost":{"currency":"USD","total_usd":0.007690000000000001,"input_uncached_usd":0.00427,"input_cached_usd":0.0,"output_usd":0.0034200000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.26628708303906}
{"item_id":"senseval2.d000.s020.t004","gold_sense_keys":["vary%2:30:04::"],"candidates":[{"index":1,"sense_key":"vary%2:30:01::","synset_id":"change.v.03"},{"index":2,"sense_key":"vary%2:30:04::","synset_id":"vary.v.04"},{"index":3,"sense_key":"vary%2:42:00::","synset_id":"deviate.v.02"},{"index":4,"sense_key":"vary%2:42:01::","synset_id":"vary.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"vary%2:30:04::","call_ids":["senseval2.d000.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"vary%2:30:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":503,"reasoning_output_tokens":487},"cost":{"currency":"USD","total_usd":0.017724999999999998,"input_uncached_usd":0.002635,"input_cached_usd":0.0,"output_usd":0.015090000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.393646790995263}
{"item_id":"senseval2.d000.s020.t005","gold_sense_keys":["order%1:14:00::"],"candidates":[{"index":1,"sense_key":"order%1:04:00::","synset_id":"order.n.15"},{"index":2,"sense_key":"order%1:10:04::","synset_id":"order.n.13"},{"index":3,"sense_key":"order%1:07:00::","synset_id":"order.n.14"},{"index":4,"sense_key":"order%1:26:01::","synset_id":"holy_order.n.01"},{"index":5,"sense_key":"order%1:07:01::","synset_id":"order.n.02"},{"index":6,"sense_key":"order%1:14:00::","synset_id":"ordering.n.01"},{"index":7,"sense_key":"order%1:10:03::","synset_id":"order.n.01"},{"index":8,"sense_key":"order%1:10:01::","synset_id":"order.n.07"},{"index":9,"sense_key":"order%1:10:00::","synset_id":"decree.n.01"},{"index":10,"sense_key":"order%1:14:02::","synset_id":"club.n.02"},{"index":11,"sense_key":"order%1:10:02::","synset_id":"order.n.09"},{"index":12,"sense_key":"order%1:14:03::","synset_id":"order.n.11"},{"index":13,"sense_key":"order%1:26:02::","synset_id":"orderliness.n.02"},{"index":14,"sense_key":"order%1:26:00::","synset_id":"order.n.03"},{"index":15,"sense_key":"order%1:14:01::","synset_id":"order.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"order%1:14:00::","call_ids":["senseval2.d000.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"order%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1069,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.008435000000000002,"input_uncached_usd":0.005345000000000001,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.735935500008054}
{"item_id":"senseval2.d000.s020.t006","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":2,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":3,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":4,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":5,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"},{"index":6,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":7,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":8,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":9,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":10,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s020.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":122},"cost":{"currency":"USD","total_usd":0.00801,"input_uncached_usd":0.00387,"input_cached_usd":0.0,"output_usd":0.0041400000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.43580458406359}
{"item_id":"senseval2.d000.s020.t007","gold_sense_keys":["sound%2:39:00::","sound%2:39:03::"],"candidates":[{"index":1,"sense_key":"sound%2:39:01::","synset_id":"sound.v.06"},{"index":2,"sense_key":"sound%2:39:00::","synset_id":"sound.v.02"},{"index":3,"sense_key":"sound%2:32:00::","synset_id":"voice.v.02"},{"index":4,"sense_key":"sound%2:30:00::","synset_id":"fathom.v.02"},{"index":5,"sense_key":"sound%2:39:03::","synset_id":"sound.v.03"},{"index":6,"sense_key":"sound%2:39:06::","synset_id":"sound.v.01"},{"index":7,"sense_key":"sound%2:39:02::","synset_id":"sound.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sound%2:39:00::","call_ids":["senseval2.d000.s020.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sound%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":575,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018715,"input_uncached_usd":0.0028750000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.961073291953653}
{"item_id":"senseval2.d000.s020.t008","gold_sense_keys":["alter%2:30:01::"],"candidates":[{"index":1,"sense_key":"alter%2:29:00::","synset_id":"alter.v.05"},{"index":2,"sense_key":"alter%2:36:00::","synset_id":"alter.v.03"},{"index":3,"sense_key":"alter%2:30:04::","synset_id":"interpolate.v.02"},{"index":4,"sense_key":"alter%2:30:00::","synset_id":"change.v.03"},{"index":5,"sense_key":"alter%2:30:01::","synset_id":"change.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"alter%2:30:01::","call_ids":["senseval2.d000.s020.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"alter%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":174,"reasoning_output_tokens":158},"cost":{"currency":"USD","total_usd":0.008105000000000001,"input_uncached_usd":0.0028850000000000004,"input_cached_usd":0.0,"output_usd":0.00522,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.690595249994658}
{"item_id":"senseval2.d000.s020.t009","gold_sense_keys":["steady%3:00:00::"],"candidates":[{"index":1,"sense_key":"steady%5:00:00:frequent:00","synset_id":"regular.s.06"},{"index":2,"sense_key":"steady%5:00:00:stable:00","synset_id":"firm.s.06"},{"index":3,"sense_key":"steady%5:00:00:unexcitable:00","synset_id":"steady.s.05"},{"index":4,"sense_key":"steady%3:00:00::","synset_id":"steady.a.01"},{"index":5,"sense_key":"steady%5:00:00:resolute:00","synset_id":"firm.s.01"},{"index":6,"sense_key":"steady%5:00:00:secure:03","synset_id":"steady.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"steady%3:00:00::","call_ids":["senseval2.d000.s020.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"steady%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.005235,"input_uncached_usd":0.0029850000000000002,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.202617875067517}
{"item_id":"senseval2.d000.s020.t010","gold_sense_keys":["rhythm%1:10:01::"],"candidates":[{"index":1,"sense_key":"rhythm%1:07:01::","synset_id":"rhythm.n.02"},{"index":2,"sense_key":"rhythm%1:10:00::","synset_id":"rhythm.n.04"},{"index":3,"sense_key":"rhythm%1:10:01::","synset_id":"rhythm.n.01"},{"index":4,"sense_key":"rhythm%1:28:00::","synset_id":"cycle.n.01"},{"index":5,"sense_key":"rhythm%1:04:00::","synset_id":"rhythm_method_of_birth_control.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"rhythm%1:10:01::","call_ids":["senseval2.d000.s020.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"rhythm%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018765,"input_uncached_usd":0.002925,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.950884209014475}
{"item_id":"senseval2.d000.s020.t011","gold_sense_keys":["striking%1:04:00::"],"candidates":[{"index":1,"sense_key":"striking%1:11:00::","synset_id":"contact.n.04"},{"index":2,"sense_key":"striking%1:04:00::","synset_id":"hit.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"striking%1:04:00::","call_ids":["senseval2.d000.s020.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"striking%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":132},"cost":{"currency":"USD","total_usd":0.006615,"input_uncached_usd":0.0022050000000000004,"input_cached_usd":0.0,"output_usd":0.00441,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.792027999996208}
{"item_id":"senseval2.d000.s021.t000","gold_sense_keys":["variation%1:10:00::"],"candidates":[{"index":1,"sense_key":"variation%1:18:00::","synset_id":"mutant.n.01"},{"index":2,"sense_key":"variation%1:22:00::","synset_id":"variation.n.07"},{"index":3,"sense_key":"variation%1:06:00::","synset_id":"variation.n.05"},{"index":4,"sense_key":"variation%1:09:01::","synset_id":"version.n.02"},{"index":5,"sense_key":"variation%1:04:02::","synset_id":"pas_seul.n.01"},{"index":6,"sense_key":"variation%1:25:00::","synset_id":"magnetic_declination.n.01"},{"index":7,"sense_key":"variation%1:11:01::","synset_id":"variation.n.01"},{"index":8,"sense_key":"variation%1:19:00::","synset_id":"variation.n.08"},{"index":9,"sense_key":"variation%1:04:00::","synset_id":"variation.n.02"},{"index":10,"sense_key":"variation%1:10:00::","synset_id":"variation.n.03"},{"index":11,"sense_key":"variation%1:04:01::","synset_id":"variation.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"variation%1:09:01::","call_ids":["senseval2.d000.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"variation%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01963,"input_uncached_usd":0.0037900000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.632102499948815}
{"item_id":"senseval2.d000.s021.t002","gold_sense_keys":["occur%2:30:00::"],"candidates":[{"index":1,"sense_key":"occur%2:31:00::","synset_id":"occur.v.02"},{"index":2,"sense_key":"occur%2:30:00::","synset_id":"happen.v.01"},{"index":3,"sense_key":"occur%2:42:03::","synset_id":"occur.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"occur%2:30:00::","call_ids":["senseval2.d000.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"occur%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01828,"input_uncached_usd":0.0024400000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.961678582942113}
{"item_id":"senseval2.d000.s021.t003","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":4,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":5,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"only%4:02:02::","call_ids":["senseval2.d000.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"only%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":107},"cost":{"currency":"USD","total_usd":0.006815,"input_uncached_usd":0.003125,"input_cached_usd":0.0,"output_usd":0.00369,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.115341415978037}
{"item_id":"senseval2.d000.s021.t004","gold_sense_keys":["once%4:02:00::"],"candidates":[{"index":1,"sense_key":"once%4:02:01::","synset_id":"once.r.03"},{"index":2,"sense_key":"once%4:02:00::","synset_id":"once.r.01"},{"index":3,"sense_key":"once%4:02:02::","synset_id":"once.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"once%4:02:00::","call_ids":["senseval2.d000.s021.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"once%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.00443,"input_uncached_usd":0.0023000000000000004,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0853361659683287}
{"item_id":"senseval2.d000.s021.t005","gold_sense_keys":["rule%1:10:00::"],"candidates":[{"index":1,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":2,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":3,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":4,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":5,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":6,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":7,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":8,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":9,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":10,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"},{"index":11,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":12,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"rule%1:10:00::","call_ids":["senseval2.d000.s021.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"rule%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":896,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02032,"input_uncached_usd":0.0044800000000000005,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.101814916008152}
{"item_id":"senseval2.d000.s021.t006","gold_sense_keys":["state%2:32:00::"],"candidates":[{"index":1,"sense_key":"state%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"state%2:32:02::","synset_id":"express.v.04"},{"index":3,"sense_key":"state%2:32:01::","synset_id":"submit.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"state%2:32:00::","call_ids":["senseval2.d000.s021.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"state%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.0037250000000000004,"input_uncached_usd":0.0023450000000000003,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0640924170147628}
{"item_id":"senseval2.d000.s022.t000","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"},{"index":2,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.004015,"input_uncached_usd":0.002455,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0589605829445645}
{"item_id":"senseval2.d000.s022.t005","gold_sense_keys":["method%1:09:00::"],"candidates":[{"index":1,"sense_key":"method%1:09:00::","synset_id":"method.n.01"},{"index":2,"sense_key":"method%1:04:00::","synset_id":"method_acting.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"method%1:09:00::","call_ids":["senseval2.d000.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"method%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0038850000000000004,"input_uncached_usd":0.0022050000000000004,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.251594458008185}
{"item_id":"senseval2.d000.s022.t006","gold_sense_keys":["name%1:10:00::"],"candidates":[{"index":1,"sense_key":"name%1:10:01::","synset_id":"name.n.06"},{"index":2,"sense_key":"name%1:26:00::","synset_id":"name.n.02"},{"index":3,"sense_key":"name%1:18:00::","synset_id":"name.n.04"},{"index":4,"sense_key":"name%1:04:00::","synset_id":"name.n.05"},{"index":5,"sense_key":"name%1:14:00::","synset_id":"name.n.03"},{"index":6,"sense_key":"name%1:10:00::","synset_id":"name.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"name%1:10:00::","call_ids":["senseval2.d000.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"name%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.0049900000000000005,"input_uncached_usd":0.00289,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.142843124922365}
{"item_id":"senseval2.d000.s023.t000","gold_sense_keys":["series%1:14:00::"],"candidates":[{"index":1,"sense_key":"series%1:14:01::","synset_id":"series.n.06"},{"index":2,"sense_key":"series%1:11:00::","synset_id":"series.n.04"},{"index":3,"sense_key":"series%1:10:00::","synset_id":"series.n.03"},{"index":4,"sense_key":"series%1:24:00::","synset_id":"series.n.05"},{"index":5,"sense_key":"series%1:10:01::","synset_id":"serial.n.01"},{"index":6,"sense_key":"series%1:14:00::","synset_id":"series.n.01"},{"index":7,"sense_key":"series%1:09:00::","synset_id":"series.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"series%1:14:00::","call_ids":["senseval2.d000.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"series%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.005445,"input_uncached_usd":0.0031950000000000004,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.465941207949072}
{"item_id":"senseval2.d000.s023.t003","gold_sense_keys":["take%2:40:06::"],"candidates":[{"index":1,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":2,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":3,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":4,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":5,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":6,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":7,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":8,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":9,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":10,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"},{"index":11,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":12,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":13,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":14,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":15,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":16,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":17,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":18,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":19,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":20,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":21,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":22,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":23,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":24,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":25,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":26,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":27,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":28,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":29,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":30,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":31,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":32,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":33,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":34,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":35,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":36,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":37,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":38,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":39,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":40,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":41,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":42,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":37,"chosen_sense_key":"take%2:40:06::","call_ids":["senseval2.d000.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":37,"predicted_sense_key":"take%2:40:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2211,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":79},"cost":{"currency":"USD","total_usd":0.013875,"input_uncached_usd":0.011055,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.345041791908443}
{"item_id":"senseval2.d000.s023.t004","gold_sense_keys":["about%4:02:00::"],"candidates":[{"index":1,"sense_key":"about%4:02:04::","synset_id":"about.r.04"},{"index":2,"sense_key":"about%4:02:02::","synset_id":"about.r.05"},{"index":3,"sense_key":"about%4:02:00::","synset_id":"approximately.r.01"},{"index":4,"sense_key":"about%4:02:05::","synset_id":"about.r.07"},{"index":5,"sense_key":"about%4:02:03::","synset_id":"about.r.06"},{"index":6,"sense_key":"about%4:02:01::","synset_id":"about.r.03"},{"index":7,"sense_key":"about%4:02:06::","synset_id":"about.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"about%4:02:00::","call_ids":["senseval2.d000.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"about%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":671,"cached_input_tokens":0,"output_tokens":42,"reasoning_output_tokens":27},"cost":{"currency":"USD","total_usd":0.004615,"input_uncached_usd":0.0033550000000000003,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7797001670114696}
{"item_id":"senseval2.d000.s023.t005","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"},{"index":2,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"},{"index":3,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"},{"index":4,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"hour%1:28:00::","call_ids":["senseval2.d000.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00429,"input_uncached_usd":0.00252,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2349254590226337}
{"item_id":"senseval2.d000.s024.t000","gold_sense_keys":["look%1:04:00::"],"candidates":[{"index":1,"sense_key":"look%1:07:01::","synset_id":"expression.n.01"},{"index":2,"sense_key":"look%1:04:00::","synset_id":"look.n.02"},{"index":3,"sense_key":"look%1:07:00::","synset_id":"look.n.03"},{"index":4,"sense_key":"look%1:26:00::","synset_id":"spirit.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"look%1:04:00::","call_ids":["senseval2.d000.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"look%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.005535,"input_uncached_usd":0.002775,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.018313250038773}
{"item_id":"senseval2.d000.s024.t002","gold_sense_keys":["night%1:28:03::"],"candidates":[{"index":1,"sense_key":"night%1:28:04::","synset_id":"night.n.03"},{"index":2,"sense_key":"night%1:28:03::","synset_id":"night.n.07"},{"index":3,"sense_key":"night%1:28:01::","synset_id":"night.n.02"},{"index":4,"sense_key":"night%1:28:00::","synset_id":"night.n.01"},{"index":5,"sense_key":"night%1:18:00::","synset_id":"nox.n.01"},{"index":6,"sense_key":"night%1:28:05::","synset_id":"night.n.06"},{"index":7,"sense_key":"night%1:26:00::","synset_id":"night.n.05"},{"index":8,"sense_key":"night%1:28:02::","synset_id":"night.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"night%1:28:03::","call_ids":["senseval2.d000.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"night%1:28:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018845,"input_uncached_usd":0.0030350000000000004,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.33305437490344}
{"item_id":"senseval2.d000.s024.t003","gold_sense_keys":["practice%1:04:02::"],"candidates":[{"index":1,"sense_key":"practice%1:04:04::","synset_id":"practice.n.03"},{"index":2,"sense_key":"practice%1:04:02::","synset_id":"exercise.n.03"},{"index":3,"sense_key":"practice%1:04:00::","synset_id":"practice.n.01"},{"index":4,"sense_key":"practice%1:04:03::","synset_id":"practice.n.04"},{"index":5,"sense_key":"practice%1:09:00::","synset_id":"practice.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"practice%1:04:02::","call_ids":["senseval2.d000.s024.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"practice%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004865,"input_uncached_usd":0.0028250000000000003,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8149062920128927}
{"item_id":"senseval2.d000.s024.t004","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:14:01::","synset_id":"church.n.04"},{"index":3,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":4,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.00461,"input_uncached_usd":0.00254,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.048130666022189}
{"item_id":"senseval2.d000.s024.t006","gold_sense_keys":["give%2:32:08::","give%2:40:03::","give%2:40:05::"],"candidates":[{"index":1,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":2,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":3,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":4,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":5,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":6,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":7,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":8,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":9,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":10,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":11,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":12,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":13,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":14,"sense_key":"give%2:29:10::","synset_id":"give.v.44"},{"index":15,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":16,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":17,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":18,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":19,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":20,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":21,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":22,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":23,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":24,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":25,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":26,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":27,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":28,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":29,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":30,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":31,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":32,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":33,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":34,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":35,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":36,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":37,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":38,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":39,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":40,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":41,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":42,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":43,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":44,"sense_key":"give%2:36:00::","synset_id":"render.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":33,"chosen_sense_key":"give%2:40:05::","call_ids":["senseval2.d000.s024.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":33,"predicted_sense_key":"give%2:40:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2106,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.026340000000000002,"input_uncached_usd":0.010530000000000001,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.688734832918271}
{"item_id":"senseval2.d000.s024.t007","gold_sense_keys":["idea%1:09:02::"],"candidates":[{"index":1,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"},{"index":2,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"},{"index":3,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"},{"index":4,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"},{"index":5,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"idea%1:09:02::","call_ids":["senseval2.d000.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"idea%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":600,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.00594,"input_uncached_usd":0.003,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6728156249737367}
{"item_id":"senseval2.d000.s024.t008","gold_sense_keys":["work%1:04:00::"],"candidates":[{"index":1,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":2,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"},{"index":3,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":4,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":5,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":6,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":7,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"work%1:04:00::","call_ids":["senseval2.d000.s024.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"work%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":717,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.006345,"input_uncached_usd":0.003585,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.238922000047751}
{"item_id":"senseval2.d000.s024.t009","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"},{"index":2,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"},{"index":3,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"},{"index":4,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"},{"index":5,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"involved%3:00:00::","call_ids":["senseval2.d000.s024.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.00507,"input_uncached_usd":0.0029700000000000004,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3329687080113217}
{"item_id":"senseval2.d000.s025.t000","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"},{"index":3,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":4,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.003885,"input_uncached_usd":0.002415,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8729182080132887}
{"item_id":"senseval2.d000.s025.t001","gold_sense_keys":["stand%2:35:00::"],"candidates":[{"index":1,"sense_key":"stand%2:35:10::","synset_id":"stand.v.12"},{"index":2,"sense_key":"stand%2:31:01::","synset_id":"stand.v.06"},{"index":3,"sense_key":"stand%2:42:02::","synset_id":"stand.v.08"},{"index":4,"sense_key":"stand%2:42:01::","synset_id":"stand.v.09"},{"index":5,"sense_key":"stand%2:35:01::","synset_id":"stand.v.10"},{"index":6,"sense_key":"stand%2:33:00::","synset_id":"resist.v.04"},{"index":7,"sense_key":"stand%2:42:06::","synset_id":"stand.v.07"},{"index":8,"sense_key":"stand%2:31:00::","synset_id":"digest.v.03"},{"index":9,"sense_key":"stand%2:35:00::","synset_id":"stand.v.01"},{"index":10,"sense_key":"stand%2:42:03::","synset_id":"stand.v.03"},{"index":11,"sense_key":"stand%2:42:00::","synset_id":"stand.v.04"},{"index":12,"sense_key":"stand%2:42:04::","synset_id":"stand.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"stand%2:35:00::","call_ids":["senseval2.d000.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"stand%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":115},"cost":{"currency":"USD","total_usd":0.007945,"input_uncached_usd":0.004015,"input_cached_usd":0.0,"output_usd":0.00393,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.293401999981143}
{"item_id":"senseval2.d000.s025.t002","gold_sense_keys":["circle%1:25:01::"],"candidates":[{"index":1,"sense_key":"circle%1:06:00::","synset_id":"circle.n.08"},{"index":2,"sense_key":"circle%1:25:00::","synset_id":"circle.n.01"},{"index":3,"sense_key":"circle%1:06:02::","synset_id":"circle.n.07"},{"index":4,"sense_key":"circle%1:06:03::","synset_id":"r-2.n.01"},{"index":5,"sense_key":"circle%1:14:00::","synset_id":"set.n.05"},{"index":6,"sense_key":"circle%1:04:00::","synset_id":"lap.n.05"},{"index":7,"sense_key":"circle%1:25:01::","synset_id":"circle.n.03"},{"index":8,"sense_key":"circle%1:06:01::","synset_id":"traffic_circle.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"circle%1:25:01::","call_ids":["senseval2.d000.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"circle%1:25:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":703,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.005435000000000001,"input_uncached_usd":0.0035150000000000003,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.063435334013775}
{"item_id":"senseval2.d000.s025.t003","gold_sense_keys":["foot%1:08:01::"],"candidates":[{"index":1,"sense_key":"foot%1:10:00::","synset_id":"metrical_foot.n.01"},{"index":2,"sense_key":"foot%1:06:00::","synset_id":"foundation.n.03"},{"index":3,"sense_key":"foot%1:05:01::","synset_id":"animal_foot.n.01"},{"index":4,"sense_key":"foot%1:14:00::","synset_id":"infantry.n.01"},{"index":5,"sense_key":"foot%1:23:00::","synset_id":"foot.n.02"},{"index":6,"sense_key":"foot%1:15:00::","synset_id":"foot.n.03"},{"index":7,"sense_key":"foot%1:05:00::","synset_id":"foot.n.06"},{"index":8,"sense_key":"foot%1:04:00::","synset_id":"foot.n.07"},{"index":9,"sense_key":"foot%1:06:01::","synset_id":"foot.n.11"},{"index":10,"sense_key":"foot%1:18:00::","synset_id":"foot.n.08"},{"index":11,"sense_key":"foot%1:08:01::","synset_id":"foot.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"foot%1:08:01::","call_ids":["senseval2.d000.s025.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"foot%1:08:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":114},"cost":{"currency":"USD","total_usd":0.007915,"input_uncached_usd":0.004015,"input_cached_usd":0.0,"output_usd":0.0039000000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.140307790949009}
{"item_id":"senseval2.d000.s025.t004","gold_sense_keys":["ahead%4:02:00::"],"candidates":[{"index":1,"sense_key":"ahead%4:02:06::","synset_id":"ahead.r.02"},{"index":2,"sense_key":"ahead%4:02:04::","synset_id":"ahead.r.06"},{"index":3,"sense_key":"ahead%4:02:00::","synset_id":"ahead.r.01"},{"index":4,"sense_key":"ahead%4:02:01::","synset_id":"ahead.r.04"},{"index":5,"sense_key":"ahead%4:02:02::","synset_id":"ahead.r.03"},{"index":6,"sense_key":"ahead%4:02:05::","synset_id":"ahead.r.05"},{"index":7,"sense_key":"ahead%4:02:03::","synset_id":"ahead.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ahead%4:02:00::","call_ids":["senseval2.d000.s025.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ahead%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.005955,"input_uncached_usd":0.003435,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4535102921072394}
{"item_id":"senseval2.d000.s025.t006","gold_sense_keys":["stance%1:07:00::"],"candidates":[{"index":1,"sense_key":"stance%1:09:00::","synset_id":"position.n.12"},{"index":2,"sense_key":"stance%1:07:00::","synset_id":"stance.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stance%1:07:00::","call_ids":["senseval2.d000.s025.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stance%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.00381,"input_uncached_usd":0.0019500000000000001,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.073569583008066}
{"item_id":"senseval2.d000.s025.t007","gold_sense_keys":["pull%2:35:04::"],"candidates":[{"index":1,"sense_key":"pull%2:38:13::","synset_id":"pull.v.09"},{"index":2,"sense_key":"pull%2:35:02::","synset_id":"attract.v.01"},{"index":3,"sense_key":"pull%2:33:13::","synset_id":"pull.v.16"},{"index":4,"sense_key":"pull%2:41:00::","synset_id":"perpetrate.v.01"},{"index":5,"sense_key":"pull%2:35:04::","synset_id":"pull.v.04"},{"index":6,"sense_key":"pull%2:35:00::","synset_id":"pull.v.01"},{"index":7,"sense_key":"pull%2:38:00::","synset_id":"pull.v.07"},{"index":8,"sense_key":"pull%2:35:10::","synset_id":"extract.v.01"},{"index":9,"sense_key":"pull%2:35:03::","synset_id":"pull.v.13"},{"index":10,"sense_key":"pull%2:29:00::","synset_id":"pull.v.08"},{"index":11,"sense_key":"pull%2:35:01::","synset_id":"rend.v.01"},{"index":12,"sense_key":"pull%2:38:01::","synset_id":"pull.v.03"},{"index":13,"sense_key":"pull%2:38:03::","synset_id":"draw.v.05"},{"index":14,"sense_key":"pull%2:38:02::","synset_id":"pull.v.10"},{"index":15,"sense_key":"pull%2:35:05::","synset_id":"pluck.v.05"},{"index":16,"sense_key":"pull%2:30:11::","synset_id":"pull.v.17"},{"index":17,"sense_key":"pull%2:38:04::","synset_id":"pull.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"pull%2:35:04::","call_ids":["senseval2.d000.s025.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"pull%2:35:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1101,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":130},"cost":{"currency":"USD","total_usd":0.009855000000000001,"input_uncached_usd":0.005505,"input_cached_usd":0.0,"output_usd":0.00435,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.662753832992166}
{"item_id":"senseval2.d000.s025.t008","gold_sense_keys":["rope%1:06:00::"],"candidates":[{"index":1,"sense_key":"rope%1:06:00::","synset_id":"rope.n.01"},{"index":2,"sense_key":"rope%1:06:01::","synset_id":"r-2.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rope%1:06:00::","call_ids":["senseval2.d000.s025.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rope%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":412,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.00353,"input_uncached_usd":0.00206,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1052039170172065}
{"item_id":"senseval2.d000.s025.t009","gold_sense_keys":["small%3:00:00::"],"candidates":[{"index":1,"sense_key":"small%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":2,"sense_key":"small%5:00:00:moderate:00","synset_id":"modest.s.01"},{"index":3,"sense_key":"small%5:00:00:young:00","synset_id":"little.s.01"},{"index":4,"sense_key":"small%5:00:00:fine:00","synset_id":"small.s.07"},{"index":5,"sense_key":"small%5:00:00:decreased:00","synset_id":"belittled.s.01"},{"index":6,"sense_key":"small%5:00:00:soft:04","synset_id":"little.s.03"},{"index":7,"sense_key":"small%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":8,"sense_key":"small%5:00:00:little:03","synset_id":"small.s.03"},{"index":9,"sense_key":"small%5:00:00:limited:00","synset_id":"minor.s.03"},{"index":10,"sense_key":"small%3:00:00::","synset_id":"small.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"small%3:00:00::","call_ids":["senseval2.d000.s025.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"small%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":783,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":112},"cost":{"currency":"USD","total_usd":0.007725,"input_uncached_usd":0.003915,"input_cached_usd":0.0,"output_usd":0.00381,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.007212583092041}
{"item_id":"senseval2.d000.s025.t010","gold_sense_keys":["hole%1:06:00::","hole%1:17:01::"],"candidates":[{"index":1,"sense_key":"hole%1:25:00::","synset_id":"hole.n.04"},{"index":2,"sense_key":"hole%1:26:00::","synset_id":"fix.n.01"},{"index":3,"sense_key":"hole%1:26:01::","synset_id":"hole.n.06"},{"index":4,"sense_key":"hole%1:17:01::","synset_id":"hole.n.01"},{"index":5,"sense_key":"hole%1:06:00::","synset_id":"hole.n.02"},{"index":6,"sense_key":"hole%1:08:00::","synset_id":"trap.n.06"},{"index":7,"sense_key":"hole%1:17:02::","synset_id":"hole.n.05"},{"index":8,"sense_key":"hole%1:06:01::","synset_id":"hole.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"hole%1:06:00::","call_ids":["senseval2.d000.s025.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"hole%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018875,"input_uncached_usd":0.0030350000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.933380875037983}
{"item_id":"senseval2.d000.s025.t011","gold_sense_keys":["high%3:00:01::"],"candidates":[{"index":1,"sense_key":"high%3:00:03::","synset_id":"high.a.04"},{"index":2,"sense_key":"high%5:00:00:superior:01","synset_id":"eminent.s.01"},{"index":3,"sense_key":"high%3:00:01::","synset_id":"high.a.02"},{"index":4,"sense_key":"high%5:00:00:malodorous:00","synset_id":"gamey.s.02"},{"index":5,"sense_key":"high%5:00:00:elated:00","synset_id":"high.s.02"},{"index":6,"sense_key":"high%5:00:00:intoxicated:00","synset_id":"high.s.04"},{"index":7,"sense_key":"high%3:00:02::","synset_id":"high.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"high%3:00:01::","call_ids":["senseval2.d000.s025.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"high%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.004615,"input_uncached_usd":0.003115,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.961922083981335}
{"item_id":"senseval2.d000.s025.t012","gold_sense_keys":["ceiling%1:06:00::"],"candidates":[{"index":1,"sense_key":"ceiling%1:07:00::","synset_id":"ceiling.n.02"},{"index":2,"sense_key":"ceiling%1:07:01::","synset_id":"ceiling.n.04"},{"index":3,"sense_key":"ceiling%1:06:00::","synset_id":"ceiling.n.01"},{"index":4,"sense_key":"ceiling%1:10:00::","synset_id":"ceiling.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ceiling%1:06:00::","call_ids":["senseval2.d000.s025.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ceiling%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00417,"input_uncached_usd":0.0024600000000000004,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.463256499962881}
{"item_id":"senseval2.d000.s025.t013","gold_sense_keys":["chamber%1:06:00::"],"candidates":[{"index":1,"sense_key":"chamber%1:06:01::","synset_id":"bedroom.n.01"},{"index":2,"sense_key":"chamber%1:08:00::","synset_id":"chamber.n.02"},{"index":3,"sense_key":"chamber%1:06:00::","synset_id":"chamber.n.01"},{"index":4,"sense_key":"chamber%1:06:02::","synset_id":"chamber.n.03"},{"index":5,"sense_key":"chamber%1:14:00::","synset_id":"chamber.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"chamber%1:06:00::","call_ids":["senseval2.d000.s025.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"chamber%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":73},"cost":{"currency":"USD","total_usd":0.005175000000000001,"input_uncached_usd":0.0025050000000000003,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.655748457997106}
{"item_id":"senseval2.d000.s026.t000","gold_sense_keys":["speak%2:32:02::"],"candidates":[{"index":1,"sense_key":"speak%2:32:01::","synset_id":"talk.v.01"},{"index":2,"sense_key":"speak%2:39:00::","synset_id":"speak.v.05"},{"index":3,"sense_key":"speak%2:32:03::","synset_id":"address.v.02"},{"index":4,"sense_key":"speak%2:32:02::","synset_id":"speak.v.03"},{"index":5,"sense_key":"speak%2:32:00::","synset_id":"talk.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"speak%2:32:02::","call_ids":["senseval2.d000.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"speak%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":540,"cached_input_tokens":0,"output_tokens":183,"reasoning_output_tokens":168},"cost":{"currency":"USD","total_usd":0.00819,"input_uncached_usd":0.0027,"input_cached_usd":0.0,"output_usd":0.00549,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.916803042055108}
{"item_id":"senseval2.d000.s026.t002","gold_sense_keys":["rope%1:06:00::"],"candidates":[{"index":1,"sense_key":"rope%1:06:01::","synset_id":"r-2.n.01"},{"index":2,"sense_key":"rope%1:06:00::","synset_id":"rope.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"rope%1:06:00::","call_ids":["senseval2.d000.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"rope%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.003445,"input_uncached_usd":0.002095,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.46863466699142}
{"item_id":"senseval2.d000.s026.t003","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval2.d000.s026.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seem%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":382,"reasoning_output_tokens":366},"cost":{"currency":"USD","total_usd":0.014135,"input_uncached_usd":0.0026750000000000003,"input_cached_usd":0.0,"output_usd":0.01146,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.337567584007047}
{"item_id":"senseval2.d000.s026.t004","gold_sense_keys":["make%2:36:08::"],"candidates":[{"index":1,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":2,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":3,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":4,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":5,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":6,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":7,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":8,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":9,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":10,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":11,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":12,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":13,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":14,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":15,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":18,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":19,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":20,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":21,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":22,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":23,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":24,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":25,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":26,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":27,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":28,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":29,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":30,"sense_key":"make%2:29:01::","synset_id":"make.v.49"},{"index":31,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":32,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":33,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":34,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":35,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":36,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":37,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":38,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":39,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":40,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":41,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":42,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":43,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":44,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":45,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":48,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":49,"sense_key":"make%2:31:10::","synset_id":"make.v.43"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":39,"chosen_sense_key":"make%2:36:00::","call_ids":["senseval2.d000.s026.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":39,"predicted_sense_key":"make%2:36:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":2308,"cached_input_tokens":0,"output_tokens":4153,"reasoning_output_tokens":4094},"cost":{"currency":"USD","total_usd":0.13613,"input_uncached_usd":0.011540000000000002,"input_cached_usd":0.0,"output_usd":0.12459,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":80.95725512504578}
{"item_id":"senseval2.d000.s026.t006","gold_sense_keys":["sound%1:07:00::"],"candidates":[{"index":1,"sense_key":"sound%1:07:00::","synset_id":"sound.n.01"},{"index":2,"sense_key":"sound%1:09:00::","synset_id":"sound.n.02"},{"index":3,"sense_key":"sound%1:17:00::","synset_id":"sound.n.08"},{"index":4,"sense_key":"sound%1:10:01::","synset_id":"phone.n.02"},{"index":5,"sense_key":"sound%1:10:00::","synset_id":"audio.n.01"},{"index":6,"sense_key":"sound%1:19:00::","synset_id":"sound.n.03"},{"index":7,"sense_key":"sound%1:11:00::","synset_id":"sound.n.04"},{"index":8,"sense_key":"sound%1:17:01::","synset_id":"strait.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sound%1:07:00::","call_ids":["senseval2.d000.s026.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sound%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":1335,"reasoning_output_tokens":1309},"cost":{"currency":"USD","total_usd":0.043375000000000004,"input_uncached_usd":0.0033250000000000003,"input_cached_usd":0.0,"output_usd":0.04005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":25.56612487498205}
{"item_id":"senseval2.d000.s026.t007","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":2,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":3,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":4,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":5,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":6,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"},{"index":7,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":8,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":9,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":10,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s026.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":779,"cached_input_tokens":0,"output_tokens":346,"reasoning_output_tokens":330},"cost":{"currency":"USD","total_usd":0.014275000000000001,"input_uncached_usd":0.0038950000000000005,"input_cached_usd":0.0,"output_usd":0.01038,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.274714582948945}
{"item_id":"senseval2.d000.s026.t008","gold_sense_keys":["muffle%2:39:00::"],"candidates":[{"index":1,"sense_key":"muffle%2:35:00::","synset_id":"smother.v.03"},{"index":2,"sense_key":"muffle%2:39:00::","synset_id":"muffle.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"muffle%2:39:00::","call_ids":["senseval2.d000.s026.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"muffle%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.00383,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8161882089916617}
{"item_id":"senseval2.d000.s026.t009","gold_sense_keys":["ceiling%1:06:00::"],"candidates":[{"index":1,"sense_key":"ceiling%1:07:00::","synset_id":"ceiling.n.02"},{"index":2,"sense_key":"ceiling%1:10:00::","synset_id":"ceiling.n.03"},{"index":3,"sense_key":"ceiling%1:07:01::","synset_id":"ceiling.n.04"},{"index":4,"sense_key":"ceiling%1:06:00::","synset_id":"ceiling.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"ceiling%1:06:00::","call_ids":["senseval2.d000.s026.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"ceiling%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.004555,"input_uncached_usd":0.0024850000000000002,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0513949170708656}
{"item_id":"senseval2.d000.s027.t001","gold_sense_keys":["absorbed%5:00:00:attentive:00"],"candidates":[{"index":1,"sense_key":"absorbed%5:00:00:unreflected:00","synset_id":"absorbed.s.02"},{"index":2,"sense_key":"absorbed%5:00:00:attentive:00","synset_id":"captive.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"absorbed%5:00:00:attentive:00","call_ids":["senseval2.d000.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"absorbed%5:00:00:attentive:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.0037500000000000003,"input_uncached_usd":0.0024000000000000002,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.025492209009826}
{"item_id":"senseval2.d000.s027.t002","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":41,"reasoning_output_tokens":26},"cost":{"currency":"USD","total_usd":0.00381,"input_uncached_usd":0.0025800000000000003,"input_cached_usd":0.0,"output_usd":0.00123,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.754955333075486}
{"item_id":"senseval2.d000.s027.t003","gold_sense_keys":["stare%2:29:00::"],"candidates":[{"index":1,"sense_key":"stare%2:39:00::","synset_id":"gaze.v.01"},{"index":2,"sense_key":"stare%2:29:00::","synset_id":"stare.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stare%2:29:00::","call_ids":["senseval2.d000.s027.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stare%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01811,"input_uncached_usd":0.0022700000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.884317166986875}
{"item_id":"senseval2.d000.s027.t004","gold_sense_keys":["straight%4:02:05::"],"candidates":[{"index":1,"sense_key":"straight%4:02:00::","synset_id":"straight.r.03"},{"index":2,"sense_key":"straight%4:02:04::","synset_id":"directly.r.04"},{"index":3,"sense_key":"straight%4:02:05::","synset_id":"directly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"straight%4:02:00::","call_ids":["senseval2.d000.s027.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"straight%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01833,"input_uncached_usd":0.00249,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.862883874913678}
{"item_id":"senseval2.d000.s027.t005","gold_sense_keys":["ahead%4:02:00::"],"candidates":[{"index":1,"sense_key":"ahead%4:02:05::","synset_id":"ahead.r.05"},{"index":2,"sense_key":"ahead%4:02:00::","synset_id":"ahead.r.01"},{"index":3,"sense_key":"ahead%4:02:04::","synset_id":"ahead.r.06"},{"index":4,"sense_key":"ahead%4:02:01::","synset_id":"ahead.r.04"},{"index":5,"sense_key":"ahead%4:02:06::","synset_id":"ahead.r.02"},{"index":6,"sense_key":"ahead%4:02:03::","synset_id":"ahead.r.07"},{"index":7,"sense_key":"ahead%4:02:02::","synset_id":"ahead.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ahead%4:02:00::","call_ids":["senseval2.d000.s027.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ahead%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":719,"cached_input_tokens":0,"output_tokens":442,"reasoning_output_tokens":426},"cost":{"currency":"USD","total_usd":0.016855,"input_uncached_usd":0.003595,"input_cached_usd":0.0,"output_usd":0.013260000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.82258466607891}
{"item_id":"senseval2.d000.s027.t006","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":2,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":3,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:34:01::","synset_id":"use.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d000.s027.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":660,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.004920000000000001,"input_uncached_usd":0.0033000000000000004,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.913476958987303}
{"item_id":"senseval2.d000.s027.t008","gold_sense_keys":["call%2:32:02::"],"candidates":[{"index":1,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":2,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":3,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":4,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":5,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":6,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":7,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":8,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":9,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":10,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":11,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":12,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":13,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":14,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":15,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":16,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":17,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":18,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":19,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":20,"sense_key":"call%2:29:05::","synset_id":"call.v.28"},{"index":21,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":22,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":23,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":24,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":25,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":26,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":27,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":28,"sense_key":"call%2:32:12::","synset_id":"call.v.19"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":16,"chosen_sense_key":"call%2:32:02::","call_ids":["senseval2.d000.s027.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":16,"predicted_sense_key":"call%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1556,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.023620000000000002,"input_uncached_usd":0.0077800000000000005,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.105194333009422}
{"item_id":"senseval2.d000.s027.t009","gold_sense_keys":["watch%2:39:00::","watch%2:39:02::","watch%2:41:00::"],"candidates":[{"index":1,"sense_key":"watch%2:32:09::","synset_id":"determine.v.08"},{"index":2,"sense_key":"watch%2:39:01::","synset_id":"watch.v.05"},{"index":3,"sense_key":"watch%2:39:00::","synset_id":"watch.v.01"},{"index":4,"sense_key":"watch%2:39:05::","synset_id":"watch.v.03"},{"index":5,"sense_key":"watch%2:41:00::","synset_id":"watch.v.02"},{"index":6,"sense_key":"watch%2:39:03::","synset_id":"watch.v.06"},{"index":7,"sense_key":"watch%2:39:02::","synset_id":"watch.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"watch%2:41:00::","call_ids":["senseval2.d000.s027.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"watch%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.034815,"input_uncached_usd":0.003435,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":21.74896754208021}
{"item_id":"senseval2.d000.s027.t010","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":2,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d000.s027.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.0053100000000000005,"input_uncached_usd":0.0027600000000000003,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5653901669429615}
{"item_id":"senseval2.d000.s027.t011","gold_sense_keys":["rope%1:06:00::"],"candidates":[{"index":1,"sense_key":"rope%1:06:01::","synset_id":"r-2.n.01"},{"index":2,"sense_key":"rope%1:06:00::","synset_id":"rope.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"rope%1:06:00::","call_ids":["senseval2.d000.s027.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"rope%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.00367,"input_uncached_usd":0.00223,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2168258749879897}
{"item_id":"senseval2.d000.s027.t012","gold_sense_keys":["thus%4:02:00::"],"candidates":[{"index":1,"sense_key":"thus%4:02:00::","synset_id":"therefore.r.01"},{"index":2,"sense_key":"thus%4:02:01::","synset_id":"thus.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"thus%4:02:00::","call_ids":["senseval2.d000.s027.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"thus%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018305000000000002,"input_uncached_usd":0.002465,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.513251208001748}
{"item_id":"senseval2.d000.s027.t013","gold_sense_keys":["time%2:30:06::"],"candidates":[{"index":1,"sense_key":"time%2:31:00::","synset_id":"time.v.02"},{"index":2,"sense_key":"time%2:31:01::","synset_id":"time.v.03"},{"index":3,"sense_key":"time%2:30:00::","synset_id":"clock.v.01"},{"index":4,"sense_key":"time%2:30:01::","synset_id":"time.v.04"},{"index":5,"sense_key":"time%2:30:06::","synset_id":"time.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"time%2:30:06::","call_ids":["senseval2.d000.s027.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"time%2:30:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":109},"cost":{"currency":"USD","total_usd":0.006645,"input_uncached_usd":0.0028950000000000004,"input_cached_usd":0.0,"output_usd":0.0037500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.371476874919608}
{"item_id":"senseval2.d000.s027.t014","gold_sense_keys":["pull%1:04:00::"],"candidates":[{"index":1,"sense_key":"pull%1:04:02::","synset_id":"pull.n.07"},{"index":2,"sense_key":"pull%1:06:01::","synset_id":"pull.n.04"},{"index":3,"sense_key":"pull%1:07:00::","synset_id":"pull.n.03"},{"index":4,"sense_key":"pull%1:04:01::","synset_id":"puff.n.07"},{"index":5,"sense_key":"pull%1:04:00::","synset_id":"pull.n.01"},{"index":6,"sense_key":"pull%1:26:00::","synset_id":"wrench.n.01"},{"index":7,"sense_key":"pull%1:19:00::","synset_id":"pull.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"pull%1:04:00::","call_ids":["senseval2.d000.s027.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"pull%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.005785,"input_uncached_usd":0.0033850000000000004,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3687671249499545}
{"item_id":"senseval2.d000.s028.t000","gold_sense_keys":["far%4:02:00::"],"candidates":[{"index":1,"sense_key":"far%4:02:01::","synset_id":"far.r.04"},{"index":2,"sense_key":"far%4:02:05::","synset_id":"far.r.05"},{"index":3,"sense_key":"far%4:02:00::","synset_id":"far.r.02"},{"index":4,"sense_key":"far%4:02:02::","synset_id":"far.r.01"},{"index":5,"sense_key":"far%4:02:04::","synset_id":"far.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"far%4:02:00::","call_ids":["senseval2.d000.s028.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"far%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":576,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.00456,"input_uncached_usd":0.00288,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9890197919448838}
{"item_id":"senseval2.d000.s028.t001","gold_sense_keys":["above%4:02:01::"],"candidates":[{"index":1,"sense_key":"above%4:02:00::","synset_id":"above.r.01"},{"index":2,"sense_key":"above%4:02:01::","synset_id":"above.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"above%4:02:01::","call_ids":["senseval2.d000.s028.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"above%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.0042250000000000005,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.073168708011508}
{"item_id":"senseval2.d000.s028.t004","gold_sense_keys":["bronze%5:00:00:metallic:00"],"candidates":[{"index":1,"sense_key":"bronze%5:00:00:chromatic:00","synset_id":"bronze.s.01"},{"index":2,"sense_key":"bronze%5:00:00:metallic:00","synset_id":"bronze.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bronze%5:00:00:metallic:00","call_ids":["senseval2.d000.s028.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bronze%5:00:00:metallic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.003795,"input_uncached_usd":0.0021750000000000003,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7576487920014188}
{"item_id":"senseval2.d000.s028.t005","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":2,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":3,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":6,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":7,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":8,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":9,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":10,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s028.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":801,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.006315000000000001,"input_uncached_usd":0.004005000000000001,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2050695419311523}
{"item_id":"senseval2.d000.s028.t006","gold_sense_keys":["mount%2:35:00::"],"candidates":[{"index":1,"sense_key":"mount%2:36:01::","synset_id":"mount.v.07"},{"index":2,"sense_key":"mount%2:38:00::","synset_id":"climb.v.01"},{"index":3,"sense_key":"mount%2:35:00::","synset_id":"mount.v.01"},{"index":4,"sense_key":"mount%2:36:00::","synset_id":"mount.v.04"},{"index":5,"sense_key":"mount%2:30:00::","synset_id":"wax.v.02"},{"index":6,"sense_key":"mount%2:38:01::","synset_id":"hop_on.v.01"},{"index":7,"sense_key":"mount%2:35:07::","synset_id":"ride.v.14"},{"index":8,"sense_key":"mount%2:30:01::","synset_id":"mount.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"mount%2:35:00::","call_ids":["senseval2.d000.s028.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"mount%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":682,"cached_input_tokens":0,"output_tokens":398,"reasoning_output_tokens":382},"cost":{"currency":"USD","total_usd":0.015349999999999999,"input_uncached_usd":0.0034100000000000003,"input_cached_usd":0.0,"output_usd":0.011940000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.931993417092599}
{"item_id":"senseval2.d000.s028.t007","gold_sense_keys":["wheel%1:06:00::"],"candidates":[{"index":1,"sense_key":"wheel%1:06:03::","synset_id":"roulette_wheel.n.01"},{"index":2,"sense_key":"wheel%1:07:00::","synset_id":"wheel.n.03"},{"index":3,"sense_key":"wheel%1:06:02::","synset_id":"rack.n.04"},{"index":4,"sense_key":"wheel%1:06:05::","synset_id":"steering_wheel.n.01"},{"index":5,"sense_key":"wheel%1:06:00::","synset_id":"wheel.n.01"},{"index":6,"sense_key":"wheel%1:06:01::","synset_id":"wheel.n.04"},{"index":7,"sense_key":"wheel%1:06:06::","synset_id":"bicycle.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"wheel%1:06:00::","call_ids":["senseval2.d000.s028.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"wheel%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":635,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.0059050000000000005,"input_uncached_usd":0.0031750000000000003,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9834563329350203}
{"item_id":"senseval2.d000.s028.t008","gold_sense_keys":["swing%2:38:03::"],"candidates":[{"index":1,"sense_key":"swing%2:38:03::","synset_id":"swing.v.03"},{"index":2,"sense_key":"swing%2:35:08::","synset_id":"swing.v.13"},{"index":3,"sense_key":"swing%2:38:00::","synset_id":"swing.v.02"},{"index":4,"sense_key":"swing%2:42:10::","synset_id":"swing.v.10"},{"index":5,"sense_key":"swing%2:41:00::","synset_id":"swing.v.11"},{"index":6,"sense_key":"swing%2:33:00::","synset_id":"swing.v.07"},{"index":7,"sense_key":"swing%2:38:02::","synset_id":"swing.v.01"},{"index":8,"sense_key":"swing%2:41:01::","synset_id":"swing.v.04"},{"index":9,"sense_key":"swing%2:35:00::","synset_id":"swing.v.05"},{"index":10,"sense_key":"swing%2:36:03::","synset_id":"swing.v.12"},{"index":11,"sense_key":"swing%2:30:08::","synset_id":"swing.v.08"},{"index":12,"sense_key":"swing%2:42:00::","synset_id":"dangle.v.01"},{"index":13,"sense_key":"swing%2:42:01::","synset_id":"swing.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"swing%2:38:03::","call_ids":["senseval2.d000.s028.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"swing%2:38:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":845,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020035,"input_uncached_usd":0.0042250000000000005,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.626088624936529}
{"item_id":"senseval2.d000.s028.t009","gold_sense_keys":["madly%4:02:02::"],"candidates":[{"index":1,"sense_key":"madly%4:02:00::","synset_id":"madly.r.03"},{"index":2,"sense_key":"madly%4:02:01::","synset_id":"insanely.r.01"},{"index":3,"sense_key":"madly%4:02:02::","synset_id":"madly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"madly%4:02:02::","call_ids":["senseval2.d000.s028.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"madly%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.0058,"input_uncached_usd":0.00256,"input_cached_usd":0.0,"output_usd":0.0032400000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.146594124962576}
{"item_id":"senseval2.d000.s028.t010","gold_sense_keys":["full%5:00:01:whole:00"],"candidates":[{"index":1,"sense_key":"full%5:00:00:high:02","synset_id":"broad.s.04"},{"index":2,"sense_key":"full%5:00:00:nourished:00","synset_id":"full.s.03"},{"index":3,"sense_key":"full%3:00:01::","synset_id":"full.a.05"},{"index":4,"sense_key":"full%5:00:02:ample:00","synset_id":"wide.s.04"},{"index":5,"sense_key":"full%5:00:01:whole:00","synset_id":"entire.s.01"},{"index":6,"sense_key":"full%5:00:00:complete:00","synset_id":"full.s.02"},{"index":7,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":8,"sense_key":"full%3:00:00::","synset_id":"full.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"full%5:00:01:whole:00","call_ids":["senseval2.d000.s028.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"full%5:00:01:whole:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":191,"reasoning_output_tokens":175},"cost":{"currency":"USD","total_usd":0.009225,"input_uncached_usd":0.0034950000000000003,"input_cached_usd":0.0,"output_usd":0.00573,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.719626542064361}
{"item_id":"senseval2.d000.s028.t011","gold_sense_keys":["degree%1:23:00::"],"candidates":[{"index":1,"sense_key":"degree%1:09:00::","synset_id":"degree.n.05"},{"index":2,"sense_key":"degree%1:07:00::","synset_id":"degree.n.01"},{"index":3,"sense_key":"degree%1:07:01::","synset_id":"degree.n.07"},{"index":4,"sense_key":"degree%1:26:01::","synset_id":"degree.n.02"},{"index":5,"sense_key":"degree%1:23:03::","synset_id":"degree.n.06"},{"index":6,"sense_key":"degree%1:23:00::","synset_id":"degree.n.04"},{"index":7,"sense_key":"degree%1:10:00::","synset_id":"academic_degree.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"degree%1:23:00::","call_ids":["senseval2.d000.s028.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"degree%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.005515000000000001,"input_uncached_usd":0.0033850000000000004,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0111677079694346}
{"item_id":"senseval2.d000.s028.t012","gold_sense_keys":["start%2:42:00::"],"candidates":[{"index":1,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":2,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":3,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":4,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":5,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":6,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":9,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":10,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":11,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":12,"sense_key":"start%2:38:03::","synset_id":"start.v.14"},{"index":13,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":14,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"start%2:42:03::","call_ids":["senseval2.d000.s028.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"start%2:42:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1001,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.036385,"input_uncached_usd":0.005005000000000001,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":18.654811750049703}
{"item_id":"senseval2.d000.s028.t013","gold_sense_keys":["end%2:42:00::"],"candidates":[{"index":1,"sense_key":"end%2:42:00::","synset_id":"end.v.01"},{"index":2,"sense_key":"end%2:42:01::","synset_id":"end.v.03"},{"index":3,"sense_key":"end%2:30:01::","synset_id":"end.v.02"},{"index":4,"sense_key":"end%2:36:13::","synset_id":"end.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"end%2:42:00::","call_ids":["senseval2.d000.s028.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"end%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":174,"reasoning_output_tokens":158},"cost":{"currency":"USD","total_usd":0.008105000000000001,"input_uncached_usd":0.0028850000000000004,"input_cached_usd":0.0,"output_usd":0.00522,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.977435625041835}
{"item_id":"senseval2.d000.s028.t015","gold_sense_keys":["inverted%5:00:00:turned:00"],"candidates":[{"index":1,"sense_key":"inverted%3:00:04::","synset_id":"anatropous.a.01"},{"index":2,"sense_key":"inverted%5:00:00:turned:00","synset_id":"inverted.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"inverted%5:00:00:turned:00","call_ids":["senseval2.d000.s028.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"inverted%5:00:00:turned:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0042250000000000005,"input_uncached_usd":0.0023350000000000003,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3072911660419777}
{"item_id":"senseval2.d000.s028.t016","gold_sense_keys":["position%1:07:00::"],"candidates":[{"index":1,"sense_key":"position%1:04:02::","synset_id":"position.n.09"},{"index":2,"sense_key":"position%1:09:00::","synset_id":"position.n.03"},{"index":3,"sense_key":"position%1:07:00::","synset_id":"position.n.07"},{"index":4,"sense_key":"position%1:09:01::","synset_id":"position.n.12"},{"index":5,"sense_key":"position%1:07:01::","synset_id":"position.n.04"},{"index":6,"sense_key":"position%1:26:00::","synset_id":"status.n.01"},{"index":7,"sense_key":"position%1:15:02::","synset_id":"military_position.n.01"},{"index":8,"sense_key":"position%1:04:03::","synset_id":"stead.n.01"},{"index":9,"sense_key":"position%1:10:01::","synset_id":"place.n.15"},{"index":10,"sense_key":"position%1:10:00::","synset_id":"side.n.10"},{"index":11,"sense_key":"position%1:04:00::","synset_id":"placement.n.03"},{"index":12,"sense_key":"position%1:15:00::","synset_id":"position.n.01"},{"index":13,"sense_key":"position%1:26:02::","synset_id":"situation.n.02"},{"index":14,"sense_key":"position%1:15:03::","synset_id":"position.n.08"},{"index":15,"sense_key":"position%1:04:04::","synset_id":"position.n.16"},{"index":16,"sense_key":"position%1:04:01::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"position%1:07:00::","call_ids":["senseval2.d000.s028.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"position%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1043,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":141},"cost":{"currency":"USD","total_usd":0.009895000000000001,"input_uncached_usd":0.005215,"input_cached_usd":0.0,"output_usd":0.00468,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.251806874992326}
{"item_id":"senseval2.d000.s029.t001","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s029.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":35,"reasoning_output_tokens":20},"cost":{"currency":"USD","total_usd":0.0036800000000000005,"input_uncached_usd":0.0026300000000000004,"input_cached_usd":0.0,"output_usd":0.00105,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":27.843997124931775}
{"item_id":"senseval2.d000.s029.t002","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":2,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":5,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":6,"sense_key":"use%2:34:01::","synset_id":"use.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d000.s029.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":670,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.00545,"input_uncached_usd":0.00335,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0636828750139102}
{"item_id":"senseval2.d000.s029.t005","gold_sense_keys":["retard%2:30:01::"],"candidates":[{"index":1,"sense_key":"retard%2:42:00::","synset_id":"retard.v.02"},{"index":2,"sense_key":"retard%2:30:07::","synset_id":"check.v.07"},{"index":3,"sense_key":"retard%2:30:00::","synset_id":"decelerate.v.01"},{"index":4,"sense_key":"retard%2:30:01::","synset_id":"retard.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"retard%2:30:01::","call_ids":["senseval2.d000.s029.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"retard%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":178,"reasoning_output_tokens":162},"cost":{"currency":"USD","total_usd":0.00802,"input_uncached_usd":0.00268,"input_cached_usd":0.0,"output_usd":0.00534,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.89583508297801}
{"item_id":"senseval2.d000.s029.t006","gold_sense_keys":["swing%1:04:00::"],"candidates":[{"index":1,"sense_key":"swing%1:04:04::","synset_id":"swing.n.09"},{"index":2,"sense_key":"swing%1:10:01::","synset_id":"swing.n.05"},{"index":3,"sense_key":"swing%1:04:06::","synset_id":"swing.n.03"},{"index":4,"sense_key":"swing%1:04:02::","synset_id":"baseball_swing.n.01"},{"index":5,"sense_key":"swing%1:26:01::","synset_id":"swing.n.01"},{"index":6,"sense_key":"swing%1:04:00::","synset_id":"swing.n.04"},{"index":7,"sense_key":"swing%1:04:03::","synset_id":"golf_stroke.n.01"},{"index":8,"sense_key":"swing%1:06:00::","synset_id":"swing.n.02"},{"index":9,"sense_key":"swing%1:07:00::","synset_id":"lilt.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"swing%1:04:00::","call_ids":["senseval2.d000.s029.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"swing%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":105},"cost":{"currency":"USD","total_usd":0.00728,"input_uncached_usd":0.0036500000000000005,"input_cached_usd":0.0,"output_usd":0.00363,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.845822250004858}
{"item_id":"senseval2.d000.s029.t007","gold_sense_keys":["so%4:02:06::"],"candidates":[{"index":1,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":2,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":3,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":4,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":5,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":6,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":7,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":8,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":9,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"},{"index":10,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"so%4:02:06::","call_ids":["senseval2.d000.s029.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"so%4:02:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":871,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":132},"cost":{"currency":"USD","total_usd":0.008795,"input_uncached_usd":0.004355,"input_cached_usd":0.0,"output_usd":0.00444,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.597353999968618}
{"item_id":"senseval2.d000.s029.t008","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":4,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":5,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":8,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":9,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s029.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":816,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.0061200000000000004,"input_uncached_usd":0.00408,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.228978375089355}
{"item_id":"senseval2.d000.s029.t009","gold_sense_keys":["swap%2:40:00::"],"candidates":[{"index":1,"sense_key":"swap%2:40:00::","synset_id":"trade.v.04"},{"index":2,"sense_key":"swap%2:38:00::","synset_id":"swap.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"swap%2:40:00::","call_ids":["senseval2.d000.s029.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"swap%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.004580000000000001,"input_uncached_usd":0.0022700000000000003,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.052688709110953}
{"item_id":"senseval2.d000.s029.t010","gold_sense_keys":["place%1:10:02::"],"candidates":[{"index":1,"sense_key":"place%1:04:00::","synset_id":"position.n.06"},{"index":2,"sense_key":"place%1:09:00::","synset_id":"place.n.03"},{"index":3,"sense_key":"place%1:15:03::","synset_id":"seat.n.01"},{"index":4,"sense_key":"place%1:10:02::","synset_id":"place.n.15"},{"index":5,"sense_key":"place%1:15:06::","synset_id":"home.n.01"},{"index":6,"sense_key":"place%1:04:01::","synset_id":"stead.n.01"},{"index":7,"sense_key":"place%1:26:01::","synset_id":"place.n.06"},{"index":8,"sense_key":"place%1:15:01::","synset_id":"place.n.04"},{"index":9,"sense_key":"place%1:15:04::","synset_id":"place.n.02"},{"index":10,"sense_key":"place%1:15:00::","synset_id":"topographic_point.n.01"},{"index":11,"sense_key":"place%1:15:02::","synset_id":"plaza.n.01"},{"index":12,"sense_key":"place%1:26:02::","synset_id":"place.n.13"},{"index":13,"sense_key":"place%1:10:01::","synset_id":"space.n.07"},{"index":14,"sense_key":"place%1:26:00::","synset_id":"place.n.10"},{"index":15,"sense_key":"place%1:10:00::","synset_id":"place.n.12"},{"index":16,"sense_key":"place%1:15:05::","synset_id":"position.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"place%1:10:02::","call_ids":["senseval2.d000.s029.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"place%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1064,"cached_input_tokens":0,"output_tokens":219,"reasoning_output_tokens":203},"cost":{"currency":"USD","total_usd":0.011890000000000001,"input_uncached_usd":0.00532,"input_cached_usd":0.0,"output_usd":0.00657,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.0147477079881355}
{"item_id":"senseval2.d000.s030.t000","gold_sense_keys":["well-known%5:00:00:known:00"],"candidates":[{"index":1,"sense_key":"well-known%5:00:00:familiar:00","synset_id":"long-familiar.s.01"},{"index":2,"sense_key":"well-known%5:00:00:known:00","synset_id":"well-known.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"well-known%5:00:00:known:00","call_ids":["senseval2.d000.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"well-known%5:00:00:known:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.00455,"input_uncached_usd":0.00236,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.406710125040263}
{"item_id":"senseval2.d000.s030.t002","gold_sense_keys":["involve%2:42:02::"],"candidates":[{"index":1,"sense_key":"involve%2:42:04::","synset_id":"involve.v.02"},{"index":2,"sense_key":"involve%2:42:00::","synset_id":"imply.v.05"},{"index":3,"sense_key":"involve%2:31:00::","synset_id":"involve.v.06"},{"index":4,"sense_key":"involve%2:42:07::","synset_id":"necessitate.v.01"},{"index":5,"sense_key":"involve%2:42:02::","synset_id":"involve.v.05"},{"index":6,"sense_key":"involve%2:30:00::","synset_id":"involve.v.07"},{"index":7,"sense_key":"involve%2:42:01::","synset_id":"involve.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"involve%2:42:02::","call_ids":["senseval2.d000.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"involve%2:42:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":651,"cached_input_tokens":0,"output_tokens":318,"reasoning_output_tokens":302},"cost":{"currency":"USD","total_usd":0.012795,"input_uncached_usd":0.003255,"input_cached_usd":0.0,"output_usd":0.00954,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.983661707956344}
{"item_id":"senseval2.d000.s030.t003","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":2,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s030.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.0047799999999999995,"input_uncached_usd":0.00268,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3790286250878125}
{"item_id":"senseval2.d000.s030.t004","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"},{"index":2,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":3,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":4,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":7,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":8,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":9,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":10,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s030.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":133},"cost":{"currency":"USD","total_usd":0.008525000000000001,"input_uncached_usd":0.0040550000000000004,"input_cached_usd":0.0,"output_usd":0.00447,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.869400291936472}
{"item_id":"senseval2.d000.s030.t005","gold_sense_keys":["english%3:01:00::"],"candidates":[{"index":1,"sense_key":"english%3:01:01::","synset_id":"english.a.02"},{"index":2,"sense_key":"english%3:01:00::","synset_id":"english.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"english%3:01:00::","call_ids":["senseval2.d000.s030.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"english%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.004495000000000001,"input_uncached_usd":0.002245,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.392311208997853}
{"item_id":"senseval2.d000.s030.t007","gold_sense_keys":["describe%2:32:00::","describe%2:32:01::"],"candidates":[{"index":1,"sense_key":"describe%2:32:01::","synset_id":"report.v.01"},{"index":2,"sense_key":"describe%2:32:00::","synset_id":"describe.v.01"},{"index":3,"sense_key":"describe%2:35:00::","synset_id":"trace.v.02"},{"index":4,"sense_key":"describe%2:31:00::","synset_id":"identify.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"describe%2:32:00::","call_ids":["senseval2.d000.s030.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"describe%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":564,"cached_input_tokens":0,"output_tokens":4125,"reasoning_output_tokens":4062},"cost":{"currency":"USD","total_usd":0.12657000000000002,"input_uncached_usd":0.00282,"input_cached_usd":0.0,"output_usd":0.12375,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":78.53107904200442}
{"item_id":"senseval2.d000.s030.t008","gold_sense_keys":["passion%1:09:00::"],"candidates":[{"index":1,"sense_key":"passion%1:12:02::","synset_id":"passion.n.05"},{"index":2,"sense_key":"passion%1:09:00::","synset_id":"love.n.02"},{"index":3,"sense_key":"passion%1:12:00::","synset_id":"passion.n.01"},{"index":4,"sense_key":"passion%1:26:01::","synset_id":"rage.n.03"},{"index":5,"sense_key":"passion%1:26:02::","synset_id":"passion.n.07"},{"index":6,"sense_key":"passion%1:16:00::","synset_id":"mania.n.01"},{"index":7,"sense_key":"passion%1:07:00::","synset_id":"heat.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"passion%1:09:00::","call_ids":["senseval2.d000.s030.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"passion%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":342,"reasoning_output_tokens":326},"cost":{"currency":"USD","total_usd":0.013345,"input_uncached_usd":0.003085,"input_cached_usd":0.0,"output_usd":0.01026,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.811203082907014}
{"item_id":"senseval2.d000.s030.t009","gold_sense_keys":["find%2:30:00::","find%2:38:10::"],"candidates":[{"index":1,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":4,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":5,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":6,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":7,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":8,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":9,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":10,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":11,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":12,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"},{"index":13,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":14,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":15,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":16,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"find%2:30:00::","call_ids":["senseval2.d000.s030.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"find%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1177,"cached_input_tokens":0,"output_tokens":255,"reasoning_output_tokens":239},"cost":{"currency":"USD","total_usd":0.013535000000000002,"input_uncached_usd":0.0058850000000000005,"input_cached_usd":0.0,"output_usd":0.0076500000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.317154709016904}
{"item_id":"senseval2.d000.s030.t010","gold_sense_keys":["satisfaction%1:12:00::","satisfaction%1:26:00::"],"candidates":[{"index":1,"sense_key":"satisfaction%1:21:01::","synset_id":"satisfaction.n.04"},{"index":2,"sense_key":"satisfaction%1:12:00::","synset_id":"satisfaction.n.01"},{"index":3,"sense_key":"satisfaction%1:04:00::","synset_id":"satisfaction.n.05"},{"index":4,"sense_key":"satisfaction%1:21:00::","synset_id":"atonement.n.01"},{"index":5,"sense_key":"satisfaction%1:26:00::","synset_id":"gratification.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"satisfaction%1:26:00::","call_ids":["senseval2.d000.s030.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"satisfaction%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01882,"input_uncached_usd":0.0029800000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.034409082960337}
{"item_id":"senseval2.d000.s030.t011","gold_sense_keys":["mathematical%5:00:00:exact:00"],"candidates":[{"index":1,"sense_key":"mathematical%5:00:00:exact:00","synset_id":"mathematical.s.03"},{"index":2,"sense_key":"mathematical%5:00:00:unquestionable:00","synset_id":"mathematical.s.01"},{"index":3,"sense_key":"mathematical%3:00:00::","synset_id":"numerical.a.03"},{"index":4,"sense_key":"mathematical%3:01:00::","synset_id":"mathematical.a.01"},{"index":5,"sense_key":"mathematical%5:00:00:possible:00","synset_id":"mathematical.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"mathematical%3:01:00::","call_ids":["senseval2.d000.s030.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"mathematical%3:01:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018770000000000002,"input_uncached_usd":0.0029300000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.525612666970119}
{"item_id":"senseval2.d000.s030.t012","gold_sense_keys":["completeness%1:26:00::"],"candidates":[{"index":1,"sense_key":"completeness%1:07:00::","synset_id":"completeness.n.02"},{"index":2,"sense_key":"completeness%1:26:00::","synset_id":"completeness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"completeness%1:26:00::","call_ids":["senseval2.d000.s030.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"completeness%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":109},"cost":{"currency":"USD","total_usd":0.006085,"input_uncached_usd":0.0023350000000000003,"input_cached_usd":0.0,"output_usd":0.0037500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.643604667042382}
{"item_id":"senseval2.d000.s030.t013","gold_sense_keys":["mechanical%3:00:00::","mechanical%3:01:00::"],"candidates":[{"index":1,"sense_key":"mechanical%3:00:00::","synset_id":"mechanical.a.01"},{"index":2,"sense_key":"mechanical%3:01:01::","synset_id":"mechanical.a.02"},{"index":3,"sense_key":"mechanical%3:01:00::","synset_id":"mechanical.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"mechanical%3:01:00::","call_ids":["senseval2.d000.s030.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"mechanical%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018405,"input_uncached_usd":0.002595,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.95163845794741}
{"item_id":"senseval2.d000.s030.t014","gold_sense_keys":["perfection%1:26:00::"],"candidates":[{"index":1,"sense_key":"perfection%1:04:00::","synset_id":"perfection.n.03"},{"index":2,"sense_key":"perfection%1:09:00::","synset_id":"paragon.n.01"},{"index":3,"sense_key":"perfection%1:26:00::","synset_id":"perfection.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"perfection%1:26:00::","call_ids":["senseval2.d000.s030.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"perfection%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.0049949999999999994,"input_uncached_usd":0.002415,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2731456250185147}
{"item_id":"senseval2.d000.s031.t000","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":2,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":3,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"},{"index":4,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.0039700000000000004,"input_uncached_usd":0.00256,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.044271875056438}
{"item_id":"senseval2.d000.s031.t001","gold_sense_keys":["add%2:32:01::"],"candidates":[{"index":1,"sense_key":"add%2:42:00::","synset_id":"add.v.06"},{"index":2,"sense_key":"add%2:32:00::","synset_id":"total.v.02"},{"index":3,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"},{"index":4,"sense_key":"add%2:31:00::","synset_id":"add.v.04"},{"index":5,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":6,"sense_key":"add%2:32:01::","synset_id":"add.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"add%2:32:01::","call_ids":["senseval2.d000.s031.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"add%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00507,"input_uncached_usd":0.0032700000000000003,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.836508166976273}
{"item_id":"senseval2.d000.s031.t002","gold_sense_keys":["filled%5:00:01:full:00"],"candidates":[{"index":1,"sense_key":"filled%5:00:00:occupied:00","synset_id":"filled.s.02"},{"index":2,"sense_key":"filled%5:00:01:full:00","synset_id":"filled.s.01"},{"index":3,"sense_key":"filled%3:44:00::","synset_id":"filled.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"filled%5:00:01:full:00","call_ids":["senseval2.d000.s031.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"filled%5:00:01:full:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.00459,"input_uncached_usd":0.0024300000000000003,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.164563832920976}
{"item_id":"senseval2.d000.s031.t003","gold_sense_keys":["solemn%5:00:00:serious:00"],"candidates":[{"index":1,"sense_key":"solemn%5:00:01:serious:00","synset_id":"earnest.s.01"},{"index":2,"sense_key":"solemn%5:00:00:serious:00","synset_id":"grave.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"solemn%5:00:00:serious:00","call_ids":["senseval2.d000.s031.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"solemn%5:00:00:serious:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.00571,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.7902344999602064}
{"item_id":"senseval2.d000.s031.t004","gold_sense_keys":["come%2:30:13::"],"candidates":[{"index":1,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":2,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":3,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":4,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":5,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":6,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":7,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":8,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":9,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":10,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":11,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":12,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":13,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":14,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":15,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":16,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":17,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":18,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":19,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":20,"sense_key":"come%2:31:13::","synset_id":"come.v.21"},{"index":21,"sense_key":"come%2:30:13::","synset_id":"come.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"come%2:42:04::","call_ids":["senseval2.d000.s031.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"come%2:42:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1249,"cached_input_tokens":0,"output_tokens":205,"reasoning_output_tokens":190},"cost":{"currency":"USD","total_usd":0.012395,"input_uncached_usd":0.0062450000000000006,"input_cached_usd":0.0,"output_usd":0.00615,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.44703841698356}
{"item_id":"senseval2.d000.s031.t006","gold_sense_keys":["ritual%1:04:00::"],"candidates":[{"index":1,"sense_key":"ritual%1:04:02::","synset_id":"ritual.n.03"},{"index":2,"sense_key":"ritual%1:04:00::","synset_id":"ritual.n.01"},{"index":3,"sense_key":"ritual%1:04:01::","synset_id":"ritual.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ritual%1:04:00::","call_ids":["senseval2.d000.s031.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ritual%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":2082,"reasoning_output_tokens":2044},"cost":{"currency":"USD","total_usd":0.064725,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.06246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":36.65991983294953}
{"item_id":"senseval2.d000.s031.t008","gold_sense_keys":["perform%2:36:00::"],"candidates":[{"index":1,"sense_key":"perform%2:41:02::","synset_id":"perform.v.02"},{"index":2,"sense_key":"perform%2:41:00::","synset_id":"do.v.03"},{"index":3,"sense_key":"perform%2:36:00::","synset_id":"perform.v.01"},{"index":4,"sense_key":"perform%2:36:01::","synset_id":"perform.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"perform%2:36:00::","call_ids":["senseval2.d000.s031.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"perform%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1206},"cost":{"currency":"USD","total_usd":0.03964000000000001,"input_uncached_usd":0.00268,"input_cached_usd":0.0,"output_usd":0.03696,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":21.842856290983036}
{"item_id":"senseval2.d000.s032.t000","gold_sense_keys":["become%2:42:01::"],"candidates":[{"index":1,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":2,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"become%2:42:01::","call_ids":["senseval2.d000.s032.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"become%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018475,"input_uncached_usd":0.002635,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.958398999995552}
{"item_id":"senseval2.d000.s032.t002","gold_sense_keys":["obsession%1:09:00::"],"candidates":[{"index":1,"sense_key":"obsession%1:16:00::","synset_id":"compulsion.n.02"},{"index":2,"sense_key":"obsession%1:09:00::","synset_id":"obsession.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"obsession%1:09:00::","call_ids":["senseval2.d000.s032.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"obsession%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":97},"cost":{"currency":"USD","total_usd":0.00561,"input_uncached_usd":0.00222,"input_cached_usd":0.0,"output_usd":0.0033900000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9054790410446003}
{"item_id":"senseval2.d000.s032.t003","gold_sense_keys":["admit%2:32:00::"],"candidates":[{"index":1,"sense_key":"admit%2:41:01::","synset_id":"admit.v.02"},{"index":2,"sense_key":"admit%2:40:00::","synset_id":"accept.v.05"},{"index":3,"sense_key":"admit%2:42:03::","synset_id":"admit.v.08"},{"index":4,"sense_key":"admit%2:42:04::","synset_id":"accommodate.v.04"},{"index":5,"sense_key":"admit%2:41:00::","synset_id":"admit.v.03"},{"index":6,"sense_key":"admit%2:42:00::","synset_id":"admit.v.06"},{"index":7,"sense_key":"admit%2:42:06::","synset_id":"admit.v.05"},{"index":8,"sense_key":"admit%2:32:00::","synset_id":"admit.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"admit%2:32:00::","call_ids":["senseval2.d000.s032.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"admit%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":751,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.005675,"input_uncached_usd":0.003755,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2382482909597456}
{"item_id":"senseval2.d000.s032.t004","gold_sense_keys":["master%1:18:03::"],"candidates":[{"index":1,"sense_key":"master%1:18:03::","synset_id":"master.n.04"},{"index":2,"sense_key":"master%1:18:02::","synset_id":"master.n.09"},{"index":3,"sense_key":"master%1:18:01::","synset_id":"victor.n.01"},{"index":4,"sense_key":"master%1:06:00::","synset_id":"master.n.06"},{"index":5,"sense_key":"master%1:18:04::","synset_id":"overlord.n.01"},{"index":6,"sense_key":"master%1:18:07::","synset_id":"master.n.07"},{"index":7,"sense_key":"master%1:18:00::","synset_id":"maestro.n.01"},{"index":8,"sense_key":"master%1:18:05::","synset_id":"master.n.08"},{"index":9,"sense_key":"master%1:18:06::","synset_id":"headmaster.n.01"},{"index":10,"sense_key":"master%1:06:01::","synset_id":"passkey.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"master%1:18:03::","call_ids":["senseval2.d000.s032.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"master%1:18:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":122},"cost":{"currency":"USD","total_usd":0.0075650000000000005,"input_uncached_usd":0.003425,"input_cached_usd":0.0,"output_usd":0.0041400000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.387224875041284}
{"item_id":"senseval2.d000.s032.t007","gold_sense_keys":["best%3:00:00::"],"candidates":[{"index":1,"sense_key":"good%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":2,"sense_key":"good%5:00:00:healthful:00","synset_id":"good.s.16"},{"index":3,"sense_key":"dear%5:00:00:close:02","synset_id":"dear.s.02"},{"index":4,"sense_key":"dependable%5:00:00:sound:00","synset_id":"dependable.s.03"},{"index":5,"sense_key":"good%5:00:00:healthy:00","synset_id":"good.s.15"},{"index":6,"sense_key":"good%5:00:00:righteous:00","synset_id":"good.s.05"},{"index":7,"sense_key":"good%5:00:00:complete:00","synset_id":"good.s.07"},{"index":8,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":9,"sense_key":"good%5:00:00:pleasing:00","synset_id":"good.s.13"},{"index":10,"sense_key":"good%5:00:00:discriminating:00","synset_id":"good.s.19"},{"index":11,"sense_key":"adept%5:00:00:skilled:00","synset_id":"adept.s.01"},{"index":12,"sense_key":"best%5:00:00:advisable:00","synset_id":"better.s.01"},{"index":13,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":14,"sense_key":"good%5:00:00:fresh:01","synset_id":"good.s.18"},{"index":15,"sense_key":"good%5:00:00:genuine:00","synset_id":"good.s.17"},{"index":16,"sense_key":"beneficial%5:00:00:advantageous:00","synset_id":"beneficial.s.01"},{"index":17,"sense_key":"best%3:00:00::","synset_id":"best.a.01"},{"index":18,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":19,"sense_key":"estimable%5:00:00:reputable:00","synset_id":"estimable.s.01"},{"index":20,"sense_key":"good%3:00:02::","synset_id":"good.a.03"},{"index":21,"sense_key":"good%5:00:00:nice:00","synset_id":"good.s.04"},{"index":22,"sense_key":"good%3:00:01::","synset_id":"good.a.01"},{"index":23,"sense_key":"good%5:00:00:opportune:00","synset_id":"good.s.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"best%3:00:00::","call_ids":["senseval2.d000.s032.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"best%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1401,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":110},"cost":{"currency":"USD","total_usd":0.010755,"input_uncached_usd":0.007005000000000001,"input_cached_usd":0.0,"output_usd":0.0037500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.692770167021081}
{"item_id":"senseval2.d000.s032.t008","gold_sense_keys":["female%3:00:00::"],"candidates":[{"index":1,"sense_key":"female%5:00:02:feminine:01","synset_id":"female.s.01"},{"index":2,"sense_key":"female%3:00:00::","synset_id":"female.a.01"},{"index":3,"sense_key":"female%5:00:01:female:00","synset_id":"female.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"female%3:00:00::","call_ids":["senseval2.d000.s032.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"female%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":337,"reasoning_output_tokens":321},"cost":{"currency":"USD","total_usd":0.012615,"input_uncached_usd":0.0025050000000000003,"input_cached_usd":0.0,"output_usd":0.010110000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.060106207965873}
{"item_id":"senseval2.d000.s032.t009","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s032.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.004235000000000001,"input_uncached_usd":0.0024950000000000003,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.926693791989237}
{"item_id":"senseval2.d000.s033.t000","gold_sense_keys":["passion%1:09:00::"],"candidates":[{"index":1,"sense_key":"passion%1:16:00::","synset_id":"mania.n.01"},{"index":2,"sense_key":"passion%1:09:00::","synset_id":"love.n.02"},{"index":3,"sense_key":"passion%1:12:02::","synset_id":"passion.n.05"},{"index":4,"sense_key":"passion%1:26:02::","synset_id":"passion.n.07"},{"index":5,"sense_key":"passion%1:26:01::","synset_id":"rage.n.03"},{"index":6,"sense_key":"passion%1:12:00::","synset_id":"passion.n.01"},{"index":7,"sense_key":"passion%1:07:00::","synset_id":"heat.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"passion%1:09:00::","call_ids":["senseval2.d000.s033.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"passion%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":172,"reasoning_output_tokens":156},"cost":{"currency":"USD","total_usd":0.00809,"input_uncached_usd":0.0029300000000000003,"input_cached_usd":0.0,"output_usd":0.0051600000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.089751166058704}
{"item_id":"senseval2.d000.s033.t002","gold_sense_keys":["stay%2:42:04::"],"candidates":[{"index":1,"sense_key":"stay%2:35:00::","synset_id":"stay.v.10"},{"index":2,"sense_key":"stay%2:30:00::","synset_id":"stay.v.01"},{"index":3,"sense_key":"stay%2:38:00::","synset_id":"stay.v.02"},{"index":4,"sense_key":"stay%2:42:02::","synset_id":"stay.v.04"},{"index":5,"sense_key":"stay%2:30:01::","synset_id":"stay.v.06"},{"index":6,"sense_key":"stay%2:41:00::","synset_id":"stay.v.09"},{"index":7,"sense_key":"stay%2:42:01::","synset_id":"bide.v.01"},{"index":8,"sense_key":"stay%2:42:04::","synset_id":"persist.v.03"},{"index":9,"sense_key":"stay%2:38:01::","synset_id":"stay.v.05"},{"index":10,"sense_key":"stay%2:42:03::","synset_id":"last_out.v.01"},{"index":11,"sense_key":"stay%2:34:00::","synset_id":"quell.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"stay%2:38:00::","call_ids":["senseval2.d000.s033.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"stay%2:38:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":801,"cached_input_tokens":0,"output_tokens":8128,"reasoning_output_tokens":8022},"cost":{"currency":"USD","total_usd":0.247845,"input_uncached_usd":0.004005000000000001,"input_cached_usd":0.0,"output_usd":0.24384,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":149.84864620794542}
{"item_id":"senseval2.d000.s033.t003","gold_sense_keys":["tower%1:06:00::"],"candidates":[{"index":1,"sense_key":"tower%1:06:00::","synset_id":"tower.n.01"},{"index":2,"sense_key":"tower%1:06:01::","synset_id":"tugboat.n.01"},{"index":3,"sense_key":"tower%1:25:00::","synset_id":"column.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tower%1:06:00::","call_ids":["senseval2.d000.s033.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tower%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.0044150000000000005,"input_uncached_usd":0.0023750000000000004,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7830169999506325}
{"item_id":"senseval2.d000.s033.t004","gold_sense_keys":["however%4:02:00::"],"candidates":[{"index":1,"sense_key":"however%4:02:00::","synset_id":"however.r.01"},{"index":2,"sense_key":"however%4:02:01::","synset_id":"however.r.04"},{"index":3,"sense_key":"however%4:02:04::","synset_id":"however.r.02"},{"index":4,"sense_key":"however%4:02:02::","synset_id":"however.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"however%4:02:00::","call_ids":["senseval2.d000.s033.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"however%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":2082,"reasoning_output_tokens":2049},"cost":{"currency":"USD","total_usd":0.06525500000000001,"input_uncached_usd":0.002795,"input_cached_usd":0.0,"output_usd":0.06246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":44.93813208397478}
{"item_id":"senseval2.d000.s034.t000","gold_sense_keys":["more%4:02:00::"],"candidates":[{"index":1,"sense_key":"more%4:02:01::","synset_id":"more.r.02"},{"index":2,"sense_key":"more%4:02:00::","synset_id":"more.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"more%4:02:00::","call_ids":["senseval2.d000.s034.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"more%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":179,"reasoning_output_tokens":163},"cost":{"currency":"USD","total_usd":0.0075000000000000015,"input_uncached_usd":0.0021300000000000004,"input_cached_usd":0.0,"output_usd":0.00537,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1498408330371603}
{"item_id":"senseval2.d000.s034.t001","gold_sense_keys":["often%4:02:00::","often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":2,"sense_key":"often%4:02:02::","synset_id":"often.r.03"},{"index":3,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"often%4:02:02::","call_ids":["senseval2.d000.s034.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"often%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":86},"cost":{"currency":"USD","total_usd":0.00532,"input_uncached_usd":0.0022600000000000003,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.821144208079204}
{"item_id":"senseval2.d000.s034.t002","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"},{"index":2,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s034.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.003985,"input_uncached_usd":0.002395,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6370641669491306}
{"item_id":"senseval2.d000.s034.t003","gold_sense_keys":["think_of%2:31:01::"],"candidates":[{"index":1,"sense_key":"think_of%2:31:02::","synset_id":"think_of.v.04"},{"index":2,"sense_key":"think_of%2:31:00::","synset_id":"remember.v.02"},{"index":3,"sense_key":"think_of%2:31:04::","synset_id":"think_of.v.06"},{"index":4,"sense_key":"think_of%2:36:00::","synset_id":"think_up.v.01"},{"index":5,"sense_key":"think_of%2:31:03::","synset_id":"entertain.v.02"},{"index":6,"sense_key":"think_of%2:31:01::","synset_id":"think_of.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"think_of%2:31:01::","call_ids":["senseval2.d000.s034.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"think_of%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":651,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.005385,"input_uncached_usd":0.003255,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9834593329578638}
{"item_id":"senseval2.d000.s034.t004","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:01::","synset_id":"church.n.04"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":4,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s034.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":90},"cost":{"currency":"USD","total_usd":0.005615,"input_uncached_usd":0.002465,"input_cached_usd":0.0,"output_usd":0.00315,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.290639500017278}
{"item_id":"senseval2.d000.s034.t006","gold_sense_keys":["bottom%1:15:01::"],"candidates":[{"index":1,"sense_key":"bottom%1:17:00::","synset_id":"bed.n.03"},{"index":2,"sense_key":"bottom%1:15:00::","synset_id":"bottom.n.01"},{"index":3,"sense_key":"bottom%1:17:01::","synset_id":"bottomland.n.01"},{"index":4,"sense_key":"bottom%1:15:01::","synset_id":"bottom.n.02"},{"index":5,"sense_key":"bottom%1:28:00::","synset_id":"bottom.n.04"},{"index":6,"sense_key":"bottom%1:06:00::","synset_id":"bottom.n.07"},{"index":7,"sense_key":"bottom%1:08:00::","synset_id":"buttocks.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bottom%1:15:00::","call_ids":["senseval2.d000.s034.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bottom%1:15:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":440,"reasoning_output_tokens":424},"cost":{"currency":"USD","total_usd":0.01634,"input_uncached_usd":0.0031400000000000004,"input_cached_usd":0.0,"output_usd":0.0132,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":15.39141254092101}
{"item_id":"senseval2.d000.s035.t001","gold_sense_keys":["complete%2:30:02::"],"candidates":[{"index":1,"sense_key":"complete%2:30:02::","synset_id":"complete.v.01"},{"index":2,"sense_key":"complete%2:41:00::","synset_id":"dispatch.v.02"},{"index":3,"sense_key":"complete%2:33:00::","synset_id":"complete.v.04"},{"index":4,"sense_key":"complete%2:30:00::","synset_id":"complete.v.02"},{"index":5,"sense_key":"complete%2:32:00::","synset_id":"complete.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"complete%2:30:02::","call_ids":["senseval2.d000.s035.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"complete%2:30:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":171,"reasoning_output_tokens":155},"cost":{"currency":"USD","total_usd":0.0076950000000000005,"input_uncached_usd":0.002565,"input_cached_usd":0.0,"output_usd":0.00513,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5585326249711215}
{"item_id":"senseval2.d000.s035.t002","gold_sense_keys":["work_up%2:29:00::"],"candidates":[{"index":1,"sense_key":"work_up%2:29:00::","synset_id":"work_up.v.02"},{"index":2,"sense_key":"work_up%2:30:06::","synset_id":"build_up.v.04"},{"index":3,"sense_key":"work_up%2:30:00::","synset_id":"build_up.v.02"},{"index":4,"sense_key":"work_up%2:30:03::","synset_id":"work_out.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"work_up%2:29:00::","call_ids":["senseval2.d000.s035.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"work_up%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.0046,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.863215749966912}
{"item_id":"senseval2.d000.s035.t003","gold_sense_keys":["sweat%1:08:00::"],"candidates":[{"index":1,"sense_key":"sweat%1:19:00::","synset_id":"sweat.n.03"},{"index":2,"sense_key":"sweat%1:04:00::","synset_id":"effort.n.02"},{"index":3,"sense_key":"sweat%1:08:00::","synset_id":"perspiration.n.01"},{"index":4,"sense_key":"sweat%1:26:01::","synset_id":"fret.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"sweat%1:08:00::","call_ids":["senseval2.d000.s035.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"sweat%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":110},"cost":{"currency":"USD","total_usd":0.006335,"input_uncached_usd":0.0025550000000000004,"input_cached_usd":0.0,"output_usd":0.00378,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.540199500042945}
{"item_id":"senseval2.d000.s035.t004","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s035.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.0038300000000000005,"input_uncached_usd":0.0023000000000000004,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2798233749344945}
{"item_id":"senseval2.d000.s035.t005","gold_sense_keys":["often%4:02:00::","often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"often%4:02:00::","call_ids":["senseval2.d000.s035.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"often%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018005,"input_uncached_usd":0.0021650000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.898835583007894}
{"item_id":"senseval2.d000.s035.t006","gold_sense_keys":["skip%2:38:02::"],"candidates":[{"index":1,"sense_key":"skip%2:38:00::","synset_id":"hop.v.01"},{"index":2,"sense_key":"skip%2:42:00::","synset_id":"cut.v.11"},{"index":3,"sense_key":"skip%2:38:03::","synset_id":"skip.v.05"},{"index":4,"sense_key":"skip%2:35:00::","synset_id":"skim.v.04"},{"index":5,"sense_key":"skip%2:38:02::","synset_id":"decamp.v.03"},{"index":6,"sense_key":"skip%2:31:01::","synset_id":"jump.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"skip%2:38:02::","call_ids":["senseval2.d000.s035.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"skip%2:38:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":310,"reasoning_output_tokens":294},"cost":{"currency":"USD","total_usd":0.011949999999999999,"input_uncached_usd":0.00265,"input_cached_usd":0.0,"output_usd":0.009300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.51618083298672}
{"item_id":"senseval2.d000.s035.t007","gold_sense_keys":["local%3:01:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:02::","synset_id":"local.a.03"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:01::","synset_id":"local.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval2.d000.s035.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004005,"input_uncached_usd":0.0022050000000000004,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5931587080704048}
{"item_id":"senseval2.d000.s035.t009","gold_sense_keys":["leave%2:40:06::"],"candidates":[{"index":1,"sense_key":"leave%2:42:01::","synset_id":"leave.v.06"},{"index":2,"sense_key":"leave%2:40:02::","synset_id":"impart.v.01"},{"index":3,"sense_key":"leave%2:30:03::","synset_id":"leave.v.12"},{"index":4,"sense_key":"leave%2:42:00::","synset_id":"leave.v.07"},{"index":5,"sense_key":"leave%2:42:02::","synset_id":"leave.v.03"},{"index":6,"sense_key":"leave%2:42:03::","synset_id":"leave.v.11"},{"index":7,"sense_key":"leave%2:38:01::","synset_id":"leave.v.01"},{"index":8,"sense_key":"leave%2:38:00::","synset_id":"exit.v.01"},{"index":9,"sense_key":"leave%2:31:05::","synset_id":"leave.v.02"},{"index":10,"sense_key":"leave%2:40:01::","synset_id":"bequeath.v.01"},{"index":11,"sense_key":"leave%2:40:06::","synset_id":"entrust.v.02"},{"index":12,"sense_key":"leave%2:30:02::","synset_id":"leave.v.04"},{"index":13,"sense_key":"leave%2:41:00::","synset_id":"leave.v.08"},{"index":14,"sense_key":"leave%2:31:02::","synset_id":"forget.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"leave%2:40:06::","call_ids":["senseval2.d000.s035.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"leave%2:40:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1003,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020855,"input_uncached_usd":0.005015,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.67206500004977}
{"item_id":"senseval2.d000.s035.t010","gold_sense_keys":["worship%1:04:00::"],"candidates":[{"index":1,"sense_key":"worship%1:12:01::","synset_id":"worship.n.02"},{"index":2,"sense_key":"worship%1:04:00::","synset_id":"worship.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"worship%1:04:00::","call_ids":["senseval2.d000.s035.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"worship%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":372,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.00393,"input_uncached_usd":0.00186,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7536321670049801}
{"item_id":"senseval2.d000.s035.t011","gold_sense_keys":["below%4:02:06::"],"candidates":[{"index":1,"sense_key":"below%4:02:06::","synset_id":"downstairs.r.01"},{"index":2,"sense_key":"below%4:02:02::","synset_id":"below.r.03"},{"index":3,"sense_key":"below%4:02:03::","synset_id":"under.r.08"},{"index":4,"sense_key":"below%4:02:00::","synset_id":"below.r.02"},{"index":5,"sense_key":"below%4:02:01::","synset_id":"below.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"below%4:02:01::","call_ids":["senseval2.d000.s035.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"below%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":277,"reasoning_output_tokens":261},"cost":{"currency":"USD","total_usd":0.010755,"input_uncached_usd":0.002445,"input_cached_usd":0.0,"output_usd":0.00831,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.328277084045112}
{"item_id":"senseval2.d000.s036.t000","gold_sense_keys":["well%4:02:00::"],"candidates":[{"index":1,"sense_key":"well%4:02:13::","synset_id":"well.r.09"},{"index":2,"sense_key":"well%4:02:12::","synset_id":"well.r.10"},{"index":3,"sense_key":"well%4:02:11::","synset_id":"well.r.07"},{"index":4,"sense_key":"well%4:02:05::","synset_id":"well.r.02"},{"index":5,"sense_key":"well%4:02:10::","synset_id":"well.r.11"},{"index":6,"sense_key":"well%4:02:08::","synset_id":"well.r.05"},{"index":7,"sense_key":"well%4:02:01::","synset_id":"well.r.13"},{"index":8,"sense_key":"well%4:02:00::","synset_id":"well.r.01"},{"index":9,"sense_key":"well%4:02:07::","synset_id":"well.r.06"},{"index":10,"sense_key":"well%4:02:09::","synset_id":"well.r.12"},{"index":11,"sense_key":"well%4:02:15::","synset_id":"well.r.04"},{"index":12,"sense_key":"well%4:02:14::","synset_id":"well.r.08"},{"index":13,"sense_key":"well%4:02:03::","synset_id":"well.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"well%4:02:01::","call_ids":["senseval2.d000.s036.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"well%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":894,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02028,"input_uncached_usd":0.00447,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.209990374976769}
{"item_id":"senseval2.d000.s037.t000","gold_sense_keys":["membership%1:14:00::"],"candidates":[{"index":1,"sense_key":"membership%1:14:00::","synset_id":"membership.n.01"},{"index":2,"sense_key":"membership%1:26:00::","synset_id":"membership.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"membership%1:14:00::","call_ids":["senseval2.d000.s037.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"membership%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":96},"cost":{"currency":"USD","total_usd":0.00542,"input_uncached_usd":0.0020900000000000003,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7564022089354694}
{"item_id":"senseval2.d000.s037.t002","gold_sense_keys":["steadily%4:02:01::"],"candidates":[{"index":1,"sense_key":"steadily%4:02:01::","synset_id":"steadily.r.01"},{"index":2,"sense_key":"steadily%4:02:00::","synset_id":"steadily.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"steadily%4:02:01::","call_ids":["senseval2.d000.s037.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"steadily%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.0041400000000000005,"input_uncached_usd":0.0020700000000000002,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1824352090479806}
{"item_id":"senseval2.d000.s037.t005","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"},{"index":2,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"},{"index":3,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s037.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.004675,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1674975419882685}
{"item_id":"senseval2.d000.s037.t006","gold_sense_keys":["press%2:32:00::"],"candidates":[{"index":1,"sense_key":"press%2:36:01::","synset_id":"press.v.07"},{"index":2,"sense_key":"press%2:36:00::","synset_id":"press.v.10"},{"index":3,"sense_key":"press%2:38:00::","synset_id":"press.v.06"},{"index":4,"sense_key":"press%2:35:02::","synset_id":"compress.v.02"},{"index":5,"sense_key":"press%2:35:03::","synset_id":"iron.v.01"},{"index":6,"sense_key":"press%2:42:00::","synset_id":"press.v.08"},{"index":7,"sense_key":"press%2:32:01::","synset_id":"bid.v.03"},{"index":8,"sense_key":"press%2:41:12::","synset_id":"crusade.v.01"},{"index":9,"sense_key":"press%2:33:00::","synset_id":"weight-lift.v.01"},{"index":10,"sense_key":"press%2:35:00::","synset_id":"press.v.01"},{"index":11,"sense_key":"press%2:42:01::","synset_id":"weigh.v.05"},{"index":12,"sense_key":"press%2:32:00::","synset_id":"urge.v.01"},{"index":13,"sense_key":"press%2:35:06::","synset_id":"press.v.11"},{"index":14,"sense_key":"press%2:35:01::","synset_id":"press.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"press%2:32:00::","call_ids":["senseval2.d000.s037.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"press%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":930,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":123},"cost":{"currency":"USD","total_usd":0.008820000000000001,"input_uncached_usd":0.0046500000000000005,"input_cached_usd":0.0,"output_usd":0.00417,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6688447089400142}
{"item_id":"senseval2.d000.s037.t007","gold_sense_keys":["equally%4:02:01::"],"candidates":[{"index":1,"sense_key":"equally%4:02:02::","synset_id":"evenly.r.01"},{"index":2,"sense_key":"equally%4:02:01::","synset_id":"equally.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"equally%4:02:01::","call_ids":["senseval2.d000.s037.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"equally%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.004815,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.959328082972206}
{"item_id":"senseval2.d000.s037.t009","gold_sense_keys":["often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"often%4:02:02::","call_ids":["senseval2.d000.s037.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"often%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.004695,"input_uncached_usd":0.002295,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.883653208031319}
{"item_id":"senseval2.d000.s037.t010","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":2,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":3,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s037.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.00408,"input_uncached_usd":0.0024300000000000003,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.125772958970629}
{"item_id":"senseval2.d000.s037.t011","gold_sense_keys":["attend%2:42:00::"],"candidates":[{"index":1,"sense_key":"attend%2:41:00::","synset_id":"serve.v.10"},{"index":2,"sense_key":"attend%2:42:01::","synset_id":"attend.v.03"},{"index":3,"sense_key":"attend%2:39:00::","synset_id":"attend.v.05"},{"index":4,"sense_key":"attend%2:41:12::","synset_id":"attend.v.02"},{"index":5,"sense_key":"attend%2:42:00::","synset_id":"attend.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"attend%2:42:00::","call_ids":["senseval2.d000.s037.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"attend%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":603,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.005775000000000001,"input_uncached_usd":0.0030150000000000003,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.027828291989863}
{"item_id":"senseval2.d000.s037.t012","gold_sense_keys":["service%1:04:01::"],"candidates":[{"index":1,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":2,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":3,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":4,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":5,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":6,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":7,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":8,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":9,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":10,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":11,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":12,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":13,"sense_key":"service%1:04:06::","synset_id":"service.n.15"},{"index":14,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":15,"sense_key":"service%1:04:04::","synset_id":"service.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"service%1:04:01::","call_ids":["senseval2.d000.s037.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"service%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":927,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.006585000000000001,"input_uncached_usd":0.004635,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.631798499962315}
{"item_id":"senseval2.d000.s038.t000","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:14:00::","synset_id":"class.n.06"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:28:01::","synset_id":"year.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d000.s038.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.0043300000000000005,"input_uncached_usd":0.00268,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9549816669896245}
{"item_id":"senseval2.d000.s038.t002","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"},{"index":2,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"},{"index":3,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s038.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0038550000000000004,"input_uncached_usd":0.0021750000000000003,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7178333339979872}
{"item_id":"senseval2.d000.s038.t003","gold_sense_keys":["get%2:30:00::"],"candidates":[{"index":1,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":2,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":3,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":4,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":5,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":6,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":7,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":8,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":9,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":10,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":11,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":12,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":13,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":14,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":15,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":16,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":18,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":19,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":20,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":21,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":22,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"},{"index":23,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":24,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":25,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":26,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":27,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":28,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":29,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":30,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":31,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":32,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":33,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":34,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":35,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":36,"sense_key":"get%2:35:12::","synset_id":"get.v.32"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":20,"chosen_sense_key":"get%2:30:00::","call_ids":["senseval2.d000.s038.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":20,"predicted_sense_key":"get%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1973,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.012865000000000001,"input_uncached_usd":0.009865,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4368267079116777}
{"item_id":"senseval2.d000.s038.t004","gold_sense_keys":["so%4:02:03::"],"candidates":[{"index":1,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":2,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":3,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":4,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":5,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":6,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"},{"index":7,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":8,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":9,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":10,"sense_key":"so%4:02:03::","synset_id":"so.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"so%4:02:03::","call_ids":["senseval2.d000.s038.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"so%4:02:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":823,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":118},"cost":{"currency":"USD","total_usd":0.008135,"input_uncached_usd":0.004115000000000001,"input_cached_usd":0.0,"output_usd":0.00402,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.1132947920123115}
{"item_id":"senseval2.d000.s038.t005","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":2,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":3,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"},{"index":4,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s038.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.0041600000000000005,"input_uncached_usd":0.00239,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4893095830921084}
{"item_id":"senseval2.d000.s038.t006","gold_sense_keys":["attend%2:42:00::"],"candidates":[{"index":1,"sense_key":"attend%2:41:12::","synset_id":"attend.v.02"},{"index":2,"sense_key":"attend%2:39:00::","synset_id":"attend.v.05"},{"index":3,"sense_key":"attend%2:41:00::","synset_id":"serve.v.10"},{"index":4,"sense_key":"attend%2:42:00::","synset_id":"attend.v.01"},{"index":5,"sense_key":"attend%2:42:01::","synset_id":"attend.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"attend%2:42:00::","call_ids":["senseval2.d000.s038.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"attend%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.0047150000000000004,"input_uncached_usd":0.002975,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6988401670241728}
{"item_id":"senseval2.d000.s038.t007","gold_sense_keys":["service%1:04:01::"],"candidates":[{"index":1,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":2,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":3,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":4,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":5,"sense_key":"service%1:04:06::","synset_id":"service.n.15"},{"index":6,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":7,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":8,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":9,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":10,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":11,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":12,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":13,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":14,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":15,"sense_key":"service%1:04:08::","synset_id":"service.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"service%1:04:01::","call_ids":["senseval2.d000.s038.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"service%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":920,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.006100000000000001,"input_uncached_usd":0.004600000000000001,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6546569999773055}
{"item_id":"senseval2.d000.s038.t008","gold_sense_keys":["sack%2:41:00::"],"candidates":[{"index":1,"sense_key":"sack%2:40:00::","synset_id":"sack.v.01"},{"index":2,"sense_key":"sack%2:41:00::","synset_id":"displace.v.03"},{"index":3,"sense_key":"sack%2:40:01::","synset_id":"net.v.01"},{"index":4,"sense_key":"sack%2:35:00::","synset_id":"sack.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sack%2:41:00::","call_ids":["senseval2.d000.s038.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sack%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.004375,"input_uncached_usd":0.002575,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5417479160241783}
{"item_id":"senseval2.d000.s038.t009","gold_sense_keys":["entire%5:00:01:whole:00"],"candidates":[{"index":1,"sense_key":"entire%5:00:02:whole:00","synset_id":"integral.s.02"},{"index":2,"sense_key":"entire%5:00:01:whole:00","synset_id":"entire.s.01"},{"index":3,"sense_key":"entire%5:00:00:smooth:02","synset_id":"entire.s.03"},{"index":4,"sense_key":"entire%5:00:00:uncastrated:00","synset_id":"entire.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"entire%5:00:01:whole:00","call_ids":["senseval2.d000.s038.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"entire%5:00:01:whole:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.004145,"input_uncached_usd":0.0026750000000000003,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5444837920367718}
{"item_id":"senseval2.d000.s038.t011","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"},{"index":2,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s038.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.00404,"input_uncached_usd":0.00239,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6461526249768212}
{"item_id":"senseval2.d000.s038.t012","gold_sense_keys":["promptly%4:02:02::"],"candidates":[{"index":1,"sense_key":"promptly%4:02:00::","synset_id":"promptly.r.02"},{"index":2,"sense_key":"promptly%4:02:01::","synset_id":"promptly.r.03"},{"index":3,"sense_key":"promptly%4:02:02::","synset_id":"promptly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"promptly%4:02:01::","call_ids":["senseval2.d000.s038.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"promptly%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01814,"input_uncached_usd":0.0023000000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.334196583018638}
{"item_id":"senseval2.d000.s038.t015","gold_sense_keys":["protest%1:04:00::"],"candidates":[{"index":1,"sense_key":"protest%1:04:00::","synset_id":"protest.n.02"},{"index":2,"sense_key":"protest%1:10:00::","synset_id":"protest.n.01"},{"index":3,"sense_key":"protest%1:10:01::","synset_id":"protest.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"protest%1:04:00::","call_ids":["senseval2.d000.s038.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"protest%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":103},"cost":{"currency":"USD","total_usd":0.005995,"input_uncached_usd":0.002425,"input_cached_usd":0.0,"output_usd":0.0035700000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0697147090686485}
{"item_id":"senseval2.d000.s039.t000","gold_sense_keys":["club%1:14:00::"],"candidates":[{"index":1,"sense_key":"club%1:06:04::","synset_id":"cabaret.n.01"},{"index":2,"sense_key":"club%1:06:03::","synset_id":"club.n.06"},{"index":3,"sense_key":"club%1:14:01::","synset_id":"baseball_club.n.01"},{"index":4,"sense_key":"club%1:06:01::","synset_id":"golf_club.n.02"},{"index":5,"sense_key":"club%1:14:00::","synset_id":"club.n.02"},{"index":6,"sense_key":"club%1:06:00::","synset_id":"club.n.03"},{"index":7,"sense_key":"club%1:06:02::","synset_id":"clubhouse.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"club%1:14:00::","call_ids":["senseval2.d000.s039.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"club%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.0055000000000000005,"input_uncached_usd":0.0034900000000000005,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5341112499590963}
{"item_id":"senseval2.d000.s039.t001","gold_sense_keys":["treat%2:31:00::"],"candidates":[{"index":1,"sense_key":"treat%2:29:00::","synset_id":"treat.v.03"},{"index":2,"sense_key":"treat%2:41:00::","synset_id":"treat.v.01"},{"index":3,"sense_key":"treat%2:34:00::","synset_id":"regale.v.01"},{"index":4,"sense_key":"treat%2:40:00::","synset_id":"treat.v.05"},{"index":5,"sense_key":"treat%2:31:00::","synset_id":"treat.v.08"},{"index":6,"sense_key":"treat%2:30:01::","synset_id":"process.v.01"},{"index":7,"sense_key":"treat%2:32:00::","synset_id":"cover.v.05"},{"index":8,"sense_key":"treat%2:32:03::","synset_id":"treat.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"treat%2:31:00::","call_ids":["senseval2.d000.s039.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"treat%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":718,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.006200000000000001,"input_uncached_usd":0.0035900000000000003,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9956151250516996}
{"item_id":"senseval2.d000.s039.t002","gold_sense_keys":["tower%1:06:00::"],"candidates":[{"index":1,"sense_key":"tower%1:25:00::","synset_id":"column.n.04"},{"index":2,"sense_key":"tower%1:06:00::","synset_id":"tower.n.01"},{"index":3,"sense_key":"tower%1:06:01::","synset_id":"tugboat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tower%1:06:00::","call_ids":["senseval2.d000.s039.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tower%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.00461,"input_uncached_usd":0.00236,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.664977957960218}
{"item_id":"senseval2.d000.s039.t003","gold_sense_keys":["sort%1:07:00::"],"candidates":[{"index":1,"sense_key":"sort%1:07:00::","synset_id":"sort.n.02"},{"index":2,"sense_key":"sort%1:22:00::","synset_id":"sort.n.04"},{"index":3,"sense_key":"sort%1:18:00::","synset_id":"sort.n.03"},{"index":4,"sense_key":"sort%1:09:00::","synset_id":"kind.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sort%1:07:00::","call_ids":["senseval2.d000.s039.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sort%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":114},"cost":{"currency":"USD","total_usd":0.00651,"input_uncached_usd":0.0026400000000000004,"input_cached_usd":0.0,"output_usd":0.00387,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.794783999910578}
{"item_id":"senseval2.d000.s039.t004","gold_sense_keys":["separate%3:00:00::","separate%5:00:00:detached:00"],"candidates":[{"index":1,"sense_key":"separate%5:00:02:segregated:00","synset_id":"separate.s.02"},{"index":2,"sense_key":"separate%3:00:00::","synset_id":"separate.a.01"},{"index":3,"sense_key":"separate%5:00:00:detached:00","synset_id":"freestanding.s.01"},{"index":4,"sense_key":"separate%5:00:02:unconnected:00","synset_id":"disjoined.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"separate%3:00:00::","call_ids":["senseval2.d000.s039.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"separate%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":523,"cached_input_tokens":0,"output_tokens":182,"reasoning_output_tokens":166},"cost":{"currency":"USD","total_usd":0.008075,"input_uncached_usd":0.002615,"input_cached_usd":0.0,"output_usd":0.0054600000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.409996625036001}
{"item_id":"senseval2.d000.s039.t006","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":2,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":3,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":4,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":5,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":6,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":7,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":8,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":9,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":10,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":11,"sense_key":"say%2:32:04::","synset_id":"order.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s039.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":829,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019985,"input_uncached_usd":0.004145,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.470909083029255}
{"item_id":"senseval2.d000.s040.t000","gold_sense_keys":["new%3:00:00::","new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":3,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"},{"index":6,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":7,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":8,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":9,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":10,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":11,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"new%5:00:00:other:00","call_ids":["senseval2.d000.s040.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"new%5:00:00:other:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":826,"cached_input_tokens":0,"output_tokens":212,"reasoning_output_tokens":196},"cost":{"currency":"USD","total_usd":0.01049,"input_uncached_usd":0.00413,"input_cached_usd":0.0,"output_usd":0.00636,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.475673750042915}
{"item_id":"senseval2.d000.s040.t002","gold_sense_keys":["today%1:28:00::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"today%1:28:00::","call_ids":["senseval2.d000.s040.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"today%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.00519,"input_uncached_usd":0.0021000000000000003,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.73992045794148}
{"item_id":"senseval2.d000.s040.t003","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":2,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":3,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"},{"index":4,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":5,"sense_key":"member%1:14:00::","synset_id":"member.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"member%1:18:00::","call_ids":["senseval2.d000.s040.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"member%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":593,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.004375,"input_uncached_usd":0.002965,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.719987459015101}
{"item_id":"senseval2.d000.s040.t004","gold_sense_keys":["congregation%1:14:00::"],"candidates":[{"index":1,"sense_key":"congregation%1:14:00::","synset_id":"congregation.n.01"},{"index":2,"sense_key":"congregation%1:04:00::","synset_id":"congregation.n.03"},{"index":3,"sense_key":"congregation%1:14:01::","synset_id":"congregation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"congregation%1:14:00::","call_ids":["senseval2.d000.s040.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"congregation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.00396,"input_uncached_usd":0.00231,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7603144579334185}
{"item_id":"senseval2.d000.s041.t000","gold_sense_keys":["still%4:02:01::"],"candidates":[{"index":1,"sense_key":"still%4:02:01::","synset_id":"still.r.01"},{"index":2,"sense_key":"still%4:02:04::","synset_id":"however.r.01"},{"index":3,"sense_key":"still%4:02:02::","synset_id":"still.r.04"},{"index":4,"sense_key":"still%4:02:00::","synset_id":"even.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"still%4:02:01::","call_ids":["senseval2.d000.s041.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"still%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.004615,"input_uncached_usd":0.002755,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.232271207962185}
{"item_id":"senseval2.d000.s041.t002","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"},{"index":4,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s041.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.003745,"input_uncached_usd":0.0023650000000000003,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5576121660415083}
{"item_id":"senseval2.d000.s041.t003","gold_sense_keys":["ring%2:39:01::"],"candidates":[{"index":1,"sense_key":"ring%2:35:03::","synset_id":"surround.v.01"},{"index":2,"sense_key":"ring%2:32:00::","synset_id":"call.v.03"},{"index":3,"sense_key":"ring%2:35:00::","synset_id":"ring.v.06"},{"index":4,"sense_key":"ring%2:39:01::","synset_id":"ring.v.03"},{"index":5,"sense_key":"ring%2:39:04::","synset_id":"resound.v.01"},{"index":6,"sense_key":"ring%2:39:00::","synset_id":"ring.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"ring%2:39:01::","call_ids":["senseval2.d000.s041.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"ring%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":113},"cost":{"currency":"USD","total_usd":0.0069500000000000004,"input_uncached_usd":0.0030800000000000003,"input_cached_usd":0.0,"output_usd":0.00387,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.018939540954307}
{"item_id":"senseval2.d000.s041.t005","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":2,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":3,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":4,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":5,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":6,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"},{"index":7,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":8,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":9,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":10,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s041.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.0058200000000000005,"input_uncached_usd":0.0038100000000000005,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.886830999981612}
{"item_id":"senseval2.d000.s042.t000","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":2,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":5,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":6,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":7,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":8,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"},{"index":9,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":10,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s042.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":778,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":123},"cost":{"currency":"USD","total_usd":0.008060000000000001,"input_uncached_usd":0.0038900000000000002,"input_cached_usd":0.0,"output_usd":0.00417,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6012248750776052}
{"item_id":"senseval2.d000.s042.t002","gold_sense_keys":["silent%5:00:00:quiet:01"],"candidates":[{"index":1,"sense_key":"silent%5:00:00:inarticulate:00","synset_id":"dumb.s.04"},{"index":2,"sense_key":"silent%5:00:00:implicit:00","synset_id":"silent.s.03"},{"index":3,"sense_key":"silent%5:00:00:uncommunicative:00","synset_id":"mum.s.01"},{"index":4,"sense_key":"silent%5:00:00:inaudible:00","synset_id":"silent.s.05"},{"index":5,"sense_key":"silent%5:00:02:inaudible:00","synset_id":"silent.s.04"},{"index":6,"sense_key":"silent%5:00:00:quiet:01","synset_id":"silent.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"silent%5:00:00:quiet:01","call_ids":["senseval2.d000.s042.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"silent%5:00:00:quiet:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":85},"cost":{"currency":"USD","total_usd":0.006105,"input_uncached_usd":0.003075,"input_cached_usd":0.0,"output_usd":0.00303,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6575207500718534}
{"item_id":"senseval2.d000.s042.t003","gold_sense_keys":["follow%2:42:01::"],"candidates":[{"index":1,"sense_key":"follow%2:36:01::","synset_id":"play_along.v.02"},{"index":2,"sense_key":"follow%2:41:09::","synset_id":"be.v.08"},{"index":3,"sense_key":"follow%2:30:00::","synset_id":"follow.v.07"},{"index":4,"sense_key":"follow%2:31:00::","synset_id":"trace.v.01"},{"index":5,"sense_key":"follow%2:42:05::","synset_id":"follow.v.03"},{"index":6,"sense_key":"follow%2:42:04::","synset_id":"come.v.05"},{"index":7,"sense_key":"follow%2:41:10::","synset_id":"follow.v.18"},{"index":8,"sense_key":"follow%2:39:13::","synset_id":"surveil.v.01"},{"index":9,"sense_key":"follow%2:41:11::","synset_id":"follow.v.19"},{"index":10,"sense_key":"follow%2:42:02::","synset_id":"follow.v.08"},{"index":11,"sense_key":"follow%2:42:03::","synset_id":"follow.v.10"},{"index":12,"sense_key":"follow%2:40:00::","synset_id":"adopt.v.01"},{"index":13,"sense_key":"follow%2:30:12::","synset_id":"stick_to.v.02"},{"index":14,"sense_key":"follow%2:41:02::","synset_id":"watch.v.02"},{"index":15,"sense_key":"follow%2:30:01::","synset_id":"keep_up.v.04"},{"index":16,"sense_key":"follow%2:36:00::","synset_id":"take_after.v.02"},{"index":17,"sense_key":"follow%2:42:00::","synset_id":"postdate.v.01"},{"index":18,"sense_key":"follow%2:41:01::","synset_id":"succeed.v.02"},{"index":19,"sense_key":"follow%2:38:13::","synset_id":"pursue.v.02"},{"index":20,"sense_key":"follow%2:38:01::","synset_id":"follow.v.04"},{"index":21,"sense_key":"follow%2:38:00::","synset_id":"follow.v.01"},{"index":22,"sense_key":"follow%2:31:14::","synset_id":"follow.v.23"},{"index":23,"sense_key":"follow%2:41:00::","synset_id":"comply.v.01"},{"index":24,"sense_key":"follow%2:42:01::","synset_id":"follow.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":24,"chosen_sense_key":"follow%2:42:01::","call_ids":["senseval2.d000.s042.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":24,"predicted_sense_key":"follow%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1397,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.013225,"input_uncached_usd":0.006985000000000001,"input_cached_usd":0.0,"output_usd":0.00624,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.338397166924551}
{"item_id":"senseval2.d000.s042.t005","gold_sense_keys":["church%1:04:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":3,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:04:00::","call_ids":["senseval2.d000.s042.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":112},"cost":{"currency":"USD","total_usd":0.00635,"input_uncached_usd":0.00251,"input_cached_usd":0.0,"output_usd":0.00384,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7222247080644593}
{"item_id":"senseval2.d000.s043.t000","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"},{"index":2,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"},{"index":3,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s043.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.005005000000000001,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.562581165926531}
{"item_id":"senseval2.d000.s043.t001","gold_sense_keys":["refuse%2:32:00::"],"candidates":[{"index":1,"sense_key":"refuse%2:41:00::","synset_id":"reject.v.06"},{"index":2,"sense_key":"refuse%2:32:00::","synset_id":"refuse.v.01"},{"index":3,"sense_key":"refuse%2:40:01::","synset_id":"deny.v.04"},{"index":4,"sense_key":"refuse%2:42:04::","synset_id":"resist.v.05"},{"index":5,"sense_key":"refuse%2:42:00::","synset_id":"defy.v.02"},{"index":6,"sense_key":"refuse%2:40:00::","synset_id":"refuse.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"refuse%2:32:00::","call_ids":["senseval2.d000.s043.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"refuse%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":80},"cost":{"currency":"USD","total_usd":0.005945000000000001,"input_uncached_usd":0.0030650000000000004,"input_cached_usd":0.0,"output_usd":0.00288,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7575700839515775}
{"item_id":"senseval2.d000.s043.t002","gold_sense_keys":["talk_about%2:32:01::"],"candidates":[{"index":1,"sense_key":"talk_about%2:32:00::","synset_id":"discourse.v.01"},{"index":2,"sense_key":"talk_about%2:32:01::","synset_id":"talk_of.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"talk_about%2:32:00::","call_ids":["senseval2.d000.s043.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"talk_about%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018015000000000003,"input_uncached_usd":0.0022050000000000004,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.147251249989495}
{"item_id":"senseval2.d000.s043.t003","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":2,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":3,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":4,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":5,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":6,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":7,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":8,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":9,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":10,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":11,"sense_key":"say%2:32:13::","synset_id":"say.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s043.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":832,"cached_input_tokens":0,"output_tokens":7605,"reasoning_output_tokens":7505},"cost":{"currency":"USD","total_usd":0.23231000000000002,"input_uncached_usd":0.0041600000000000005,"input_cached_usd":0.0,"output_usd":0.22815000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":125.29968416597694}
{"item_id":"senseval2.d000.s044.t000","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"},{"index":2,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"},{"index":3,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s044.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":429,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.004515,"input_uncached_usd":0.002145,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1704733329825103}
{"item_id":"senseval2.d000.s044.t002","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:14:01::","synset_id":"church.n.04"},{"index":4,"sense_key":"church%1:14:00::","synset_id":"church.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s044.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":371,"reasoning_output_tokens":355},"cost":{"currency":"USD","total_usd":0.013555,"input_uncached_usd":0.002425,"input_cached_usd":0.0,"output_usd":0.011130000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.506163875106722}
{"item_id":"senseval2.d000.s044.t003","gold_sense_keys":["feel%2:31:00::"],"candidates":[{"index":1,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":2,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":3,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":4,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"},{"index":5,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":6,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":7,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":8,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":9,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":10,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":13,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"feel%2:31:00::","call_ids":["senseval2.d000.s044.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"feel%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":880,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":81},"cost":{"currency":"USD","total_usd":0.0073100000000000005,"input_uncached_usd":0.0044,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2858915829565376}
{"item_id":"senseval2.d000.s044.t004","gold_sense_keys":["fault%1:07:01::"],"candidates":[{"index":1,"sense_key":"fault%1:04:00::","synset_id":"mistake.n.01"},{"index":2,"sense_key":"fault%1:17:00::","synset_id":"fault.n.04"},{"index":3,"sense_key":"fault%1:11:00::","synset_id":"fault.n.05"},{"index":4,"sense_key":"fault%1:07:01::","synset_id":"fault.n.06"},{"index":5,"sense_key":"fault%1:07:00::","synset_id":"demerit.n.02"},{"index":6,"sense_key":"fault%1:26:00::","synset_id":"defect.n.03"},{"index":7,"sense_key":"fault%1:04:01::","synset_id":"fault.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"fault%1:26:00::","call_ids":["senseval2.d000.s044.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"fault%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":705,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019365,"input_uncached_usd":0.0035250000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.086421207990497}
{"item_id":"senseval2.d000.s044.t008","gold_sense_keys":["next%5:00:00:close:01"],"candidates":[{"index":1,"sense_key":"next%5:00:00:incoming:00","synset_id":"future.s.02"},{"index":2,"sense_key":"next%5:00:00:succeeding:00","synset_id":"following.s.02"},{"index":3,"sense_key":"next%5:00:00:close:01","synset_id":"adjacent.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"next%5:00:00:close:01","call_ids":["senseval2.d000.s044.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"next%5:00:00:close:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":42,"reasoning_output_tokens":27},"cost":{"currency":"USD","total_usd":0.0036050000000000006,"input_uncached_usd":0.0023450000000000003,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.816083375015296}
{"item_id":"senseval2.d000.s044.t009","gold_sense_keys":["altar%1:06:01::"],"candidates":[{"index":1,"sense_key":"altar%1:06:00::","synset_id":"altar.n.02"},{"index":2,"sense_key":"altar%1:06:01::","synset_id":"altar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"altar%1:06:01::","call_ids":["senseval2.d000.s044.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"altar%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":398,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004030000000000001,"input_uncached_usd":0.00199,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9336678750114515}
{"item_id":"senseval2.d000.s045.t000","gold_sense_keys":["so%4:02:10::"],"candidates":[{"index":1,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"},{"index":2,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":3,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":4,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":5,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":6,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":7,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":8,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":9,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":10,"sense_key":"so%4:02:08::","synset_id":"so.r.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"so%4:02:10::","call_ids":["senseval2.d000.s045.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"so%4:02:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":814,"cached_input_tokens":0,"output_tokens":289,"reasoning_output_tokens":274},"cost":{"currency":"USD","total_usd":0.012740000000000001,"input_uncached_usd":0.004070000000000001,"input_cached_usd":0.0,"output_usd":0.00867,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.134665166027844}
{"item_id":"senseval2.d000.s045.t007","gold_sense_keys":["here%4:02:01::"],"candidates":[{"index":1,"sense_key":"here%4:02:00::","synset_id":"here.r.01"},{"index":2,"sense_key":"here%4:02:02::","synset_id":"here.r.02"},{"index":3,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":4,"sense_key":"here%4:02:04::","synset_id":"here.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"here%4:02:01::","call_ids":["senseval2.d000.s045.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"here%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01827,"input_uncached_usd":0.0024300000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.071776166907512}
{"item_id":"senseval2.d000.s045.t008","gold_sense_keys":["come%2:38:00::","come%2:38:04::"],"candidates":[{"index":1,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":2,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":3,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":4,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":5,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":6,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":7,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":8,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":9,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":10,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":11,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":12,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":13,"sense_key":"come%2:31:13::","synset_id":"come.v.21"},{"index":14,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":15,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":16,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":17,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":18,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":19,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":20,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":21,"sense_key":"come%2:42:04::","synset_id":"come.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"come%2:38:00::","call_ids":["senseval2.d000.s045.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"come%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1205,"cached_input_tokens":0,"output_tokens":360,"reasoning_output_tokens":344},"cost":{"currency":"USD","total_usd":0.016825,"input_uncached_usd":0.006025000000000001,"input_cached_usd":0.0,"output_usd":0.0108,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.405459875008091}
{"item_id":"senseval2.d000.s045.t009","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":2,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":3,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"},{"index":4,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s045.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.00424,"input_uncached_usd":0.00235,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7490194999845698}
{"item_id":"senseval2.d000.s045.t010","gold_sense_keys":["above%4:02:01::"],"candidates":[{"index":1,"sense_key":"above%4:02:01::","synset_id":"above.r.02"},{"index":2,"sense_key":"above%4:02:00::","synset_id":"above.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"above%4:02:01::","call_ids":["senseval2.d000.s045.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"above%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.00413,"input_uncached_usd":0.002,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9894248340278864}
{"item_id":"senseval2.d000.s045.t011","gold_sense_keys":["make%2:36:12::"],"candidates":[{"index":1,"sense_key":"make%2:29:01::","synset_id":"make.v.49"},{"index":2,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":3,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":4,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":5,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":6,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":7,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":8,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":9,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":10,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":11,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":12,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":13,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":14,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":15,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":16,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":17,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":18,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":19,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":20,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":21,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":22,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":23,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":24,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":25,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":26,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":27,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":28,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":29,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":30,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":31,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":32,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":33,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":34,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":35,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":36,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":37,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":38,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":39,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":40,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":41,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":42,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":43,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":46,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":47,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":48,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":49,"sense_key":"make%2:42:11::","synset_id":"make.v.33"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"make%2:36:12::","call_ids":["senseval2.d000.s045.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"make%2:36:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2289,"cached_input_tokens":0,"output_tokens":309,"reasoning_output_tokens":293},"cost":{"currency":"USD","total_usd":0.020715,"input_uncached_usd":0.011445,"input_cached_usd":0.0,"output_usd":0.00927,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.8699358749436215}
{"item_id":"senseval2.d000.s045.t012","gold_sense_keys":["very%4:02:00::"],"candidates":[{"index":1,"sense_key":"very%4:02:00::","synset_id":"very.r.01"},{"index":2,"sense_key":"very%4:02:01::","synset_id":"very.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"very%4:02:00::","call_ids":["senseval2.d000.s045.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"very%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":429,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.004635,"input_uncached_usd":0.002145,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.178855915903114}
{"item_id":"senseval2.d000.s045.t014","gold_sense_keys":["exit%1:04:00::"],"candidates":[{"index":1,"sense_key":"exit%1:06:00::","synset_id":"exit.n.01"},{"index":2,"sense_key":"exit%1:04:00::","synset_id":"exit.n.03"},{"index":3,"sense_key":"exit%1:11:00::","synset_id":"passing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"exit%1:04:00::","call_ids":["senseval2.d000.s045.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"exit%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.0044,"input_uncached_usd":0.0022400000000000002,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4976128749549389}
{"item_id":"senseval2.d000.s045.t015","gold_sense_keys":["congregation%1:14:00::"],"candidates":[{"index":1,"sense_key":"congregation%1:14:00::","synset_id":"congregation.n.01"},{"index":2,"sense_key":"congregation%1:14:01::","synset_id":"congregation.n.02"},{"index":3,"sense_key":"congregation%1:04:00::","synset_id":"congregation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"congregation%1:14:00::","call_ids":["senseval2.d000.s045.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"congregation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.004935,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.517306499998085}
{"item_id":"senseval2.d000.s045.t016","gold_sense_keys":["prayer%1:04:00::"],"candidates":[{"index":1,"sense_key":"prayer%1:10:01::","synset_id":"prayer.n.04"},{"index":2,"sense_key":"prayer%1:04:00::","synset_id":"prayer.n.01"},{"index":3,"sense_key":"prayer%1:18:00::","synset_id":"prayer.n.05"},{"index":4,"sense_key":"prayer%1:10:00::","synset_id":"entreaty.n.01"},{"index":5,"sense_key":"prayer%1:10:02::","synset_id":"prayer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"prayer%1:04:00::","call_ids":["senseval2.d000.s045.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"prayer%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.00541,"input_uncached_usd":0.0025900000000000003,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4649254999822006}
{"item_id":"senseval2.d000.s045.t017","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":3,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":4,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":5,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":6,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":7,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":8,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":9,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":10,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":11,"sense_key":"say%2:32:15::","synset_id":"say.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s045.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":814,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01991,"input_uncached_usd":0.004070000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.68110941699706}
{"item_id":"senseval2.d000.s046.t000","gold_sense_keys":["admit%2:32:00::"],"candidates":[{"index":1,"sense_key":"admit%2:42:00::","synset_id":"admit.v.06"},{"index":2,"sense_key":"admit%2:41:01::","synset_id":"admit.v.02"},{"index":3,"sense_key":"admit%2:42:06::","synset_id":"admit.v.05"},{"index":4,"sense_key":"admit%2:42:04::","synset_id":"accommodate.v.04"},{"index":5,"sense_key":"admit%2:42:03::","synset_id":"admit.v.08"},{"index":6,"sense_key":"admit%2:32:00::","synset_id":"admit.v.01"},{"index":7,"sense_key":"admit%2:40:00::","synset_id":"accept.v.05"},{"index":8,"sense_key":"admit%2:41:00::","synset_id":"admit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"admit%2:32:00::","call_ids":["senseval2.d000.s046.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"admit%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":722,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00553,"input_uncached_usd":0.0036100000000000004,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8465503330808133}
{"item_id":"senseval2.d000.s046.t001","gold_sense_keys":["mixed%5:00:00:heterogeneous:00"],"candidates":[{"index":1,"sense_key":"mixed%5:00:00:integrated:00","synset_id":"interracial.s.02"},{"index":2,"sense_key":"mixed%5:00:00:heterogeneous:00","synset_id":"assorted.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mixed%5:00:00:heterogeneous:00","call_ids":["senseval2.d000.s046.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mixed%5:00:00:heterogeneous:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.004635,"input_uncached_usd":0.0022050000000000004,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5597167499363422}
{"item_id":"senseval2.d000.s046.t002","gold_sense_keys":["feeling%1:03:00::","feeling%1:09:03::"],"candidates":[{"index":1,"sense_key":"feeling%1:09:02::","synset_id":"touch.n.10"},{"index":2,"sense_key":"feeling%1:26:00::","synset_id":"spirit.n.02"},{"index":3,"sense_key":"feeling%1:09:01::","synset_id":"feeling.n.04"},{"index":4,"sense_key":"feeling%1:03:00::","synset_id":"feeling.n.01"},{"index":5,"sense_key":"feeling%1:09:00::","synset_id":"feeling.n.06"},{"index":6,"sense_key":"feeling%1:09:03::","synset_id":"impression.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"feeling%1:03:00::","call_ids":["senseval2.d000.s046.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"feeling%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":1540,"reasoning_output_tokens":1512},"cost":{"currency":"USD","total_usd":0.04939,"input_uncached_usd":0.00319,"input_cached_usd":0.0,"output_usd":0.0462,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":25.54234245896805}
{"item_id":"senseval2.d000.s046.t003","gold_sense_keys":["issue%1:09:00::","issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":2,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":5,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":6,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"},{"index":7,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":8,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":11,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"issue%1:09:01::","call_ids":["senseval2.d000.s046.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":873,"cached_input_tokens":0,"output_tokens":240,"reasoning_output_tokens":224},"cost":{"currency":"USD","total_usd":0.011565,"input_uncached_usd":0.004365,"input_cached_usd":0.0,"output_usd":0.0072,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.024467041017488}
{"item_id":"senseval2.d000.s046.t004","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"},{"index":2,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"},{"index":3,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s046.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.004710000000000001,"input_uncached_usd":0.0021300000000000004,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.224202624987811}
{"item_id":"senseval2.d000.s046.t005","gold_sense_keys":["active%5:00:00:involved:00"],"candidates":[{"index":1,"sense_key":"active%3:00:06::","synset_id":"active.a.07"},{"index":2,"sense_key":"active%3:00:05::","synset_id":"active.a.12"},{"index":3,"sense_key":"active%3:00:08::","synset_id":"active.a.14"},{"index":4,"sense_key":"active%3:00:09::","synset_id":"active.a.10"},{"index":5,"sense_key":"active%5:00:00:existent:00","synset_id":"active.s.03"},{"index":6,"sense_key":"active%3:00:03::","synset_id":"active.a.03"},{"index":7,"sense_key":"active%3:00:04::","synset_id":"active.a.13"},{"index":8,"sense_key":"active%5:00:00:operational:00","synset_id":"active.s.01"},{"index":9,"sense_key":"active%3:00:01::","synset_id":"active.a.05"},{"index":10,"sense_key":"active%3:00:07::","synset_id":"active.a.06"},{"index":11,"sense_key":"active%5:00:00:involved:00","synset_id":"active.s.02"},{"index":12,"sense_key":"active%3:00:00::","synset_id":"active.a.11"},{"index":13,"sense_key":"active%3:00:10::","synset_id":"active.a.09"},{"index":14,"sense_key":"active%3:00:02::","synset_id":"active.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"active%5:00:00:involved:00","call_ids":["senseval2.d000.s046.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"active%5:00:00:involved:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":877,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":108},"cost":{"currency":"USD","total_usd":0.008105000000000001,"input_uncached_usd":0.004385,"input_cached_usd":0.0,"output_usd":0.00372,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6376863330369815}
{"item_id":"senseval2.d000.s046.t006","gold_sense_keys":["bell_ringer%1:18:01::"],"candidates":[{"index":1,"sense_key":"bell_ringer%1:18:00::","synset_id":"bell_ringer.n.02"},{"index":2,"sense_key":"bell_ringer%1:04:00::","synset_id":"bell_ringer.n.03"},{"index":3,"sense_key":"bell_ringer%1:18:01::","synset_id":"toller.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bell_ringer%1:18:01::","call_ids":["senseval2.d000.s046.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bell_ringer%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.003625,"input_uncached_usd":0.002275,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4106890419498086}
{"item_id":"senseval2.d000.s047.t000","gold_sense_keys":["sound%1:07:00::"],"candidates":[{"index":1,"sense_key":"sound%1:17:00::","synset_id":"sound.n.08"},{"index":2,"sense_key":"sound%1:07:00::","synset_id":"sound.n.01"},{"index":3,"sense_key":"sound%1:10:01::","synset_id":"phone.n.02"},{"index":4,"sense_key":"sound%1:17:01::","synset_id":"strait.n.01"},{"index":5,"sense_key":"sound%1:11:00::","synset_id":"sound.n.04"},{"index":6,"sense_key":"sound%1:10:00::","synset_id":"audio.n.01"},{"index":7,"sense_key":"sound%1:19:00::","synset_id":"sound.n.03"},{"index":8,"sense_key":"sound%1:09:00::","synset_id":"sound.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sound%1:07:00::","call_ids":["senseval2.d000.s047.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sound%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":646,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.005180000000000001,"input_uncached_usd":0.0032300000000000002,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.527134875068441}
{"item_id":"senseval2.d000.s047.t001","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":2,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":3,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":4,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":5,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":8,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"},{"index":9,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":10,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s047.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":760,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01964,"input_uncached_usd":0.0038000000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.885027959011495}
{"item_id":"senseval2.d000.s047.t002","gold_sense_keys":["draw%2:35:06::"],"candidates":[{"index":1,"sense_key":"draw%2:38:07::","synset_id":"draw.v.17"},{"index":2,"sense_key":"draw%2:37:01::","synset_id":"draw.v.10"},{"index":3,"sense_key":"draw%2:33:00::","synset_id":"tie.v.02"},{"index":4,"sense_key":"draw%2:38:11::","synset_id":"pull.v.09"},{"index":5,"sense_key":"draw%2:35:02::","synset_id":"draw.v.20"},{"index":6,"sense_key":"draw%2:36:01::","synset_id":"draw.v.15"},{"index":7,"sense_key":"draw%2:36:07::","synset_id":"draw.v.18"},{"index":8,"sense_key":"draw%2:40:00::","synset_id":"reap.v.02"},{"index":9,"sense_key":"draw%2:30:14::","synset_id":"draw.v.16"},{"index":10,"sense_key":"draw%2:38:01::","synset_id":"draw.v.05"},{"index":11,"sense_key":"draw%2:35:06::","synset_id":"attract.v.01"},{"index":12,"sense_key":"draw%2:40:01::","synset_id":"withdraw.v.09"},{"index":13,"sense_key":"draw%2:31:13::","synset_id":"draw.v.09"},{"index":14,"sense_key":"draw%2:35:09::","synset_id":"pull_back.v.04"},{"index":15,"sense_key":"draw%2:29:00::","synset_id":"draw.v.36"},{"index":16,"sense_key":"draw%2:32:00::","synset_id":"describe.v.01"},{"index":17,"sense_key":"draw%2:31:00::","synset_id":"draw.v.04"},{"index":18,"sense_key":"draw%2:42:00::","synset_id":"draw.v.21"},{"index":19,"sense_key":"draw%2:36:02::","synset_id":"draw.v.19"},{"index":20,"sense_key":"draw%2:41:13::","synset_id":"draw.v.23"},{"index":21,"sense_key":"draw%2:38:00::","synset_id":"draw.v.07"},{"index":22,"sense_key":"draw%2:35:03::","synset_id":"pull.v.01"},{"index":23,"sense_key":"draw%2:42:02::","synset_id":"draw.v.22"},{"index":24,"sense_key":"draw%2:35:13::","synset_id":"guide.v.05"},{"index":25,"sense_key":"draw%2:30:00::","synset_id":"draw.v.31"},{"index":26,"sense_key":"draw%2:35:01::","synset_id":"trace.v.02"},{"index":27,"sense_key":"draw%2:30:04::","synset_id":"draw.v.35"},{"index":28,"sense_key":"draw%2:35:04::","synset_id":"absorb.v.04"},{"index":29,"sense_key":"draw%2:30:05::","synset_id":"disembowel.v.01"},{"index":30,"sense_key":"draw%2:30:15::","synset_id":"draw.v.33"},{"index":31,"sense_key":"draw%2:35:14::","synset_id":"string.v.01"},{"index":32,"sense_key":"draw%2:36:00::","synset_id":"draw.v.06"},{"index":33,"sense_key":"draw%2:33:01::","synset_id":"draw.v.14"},{"index":34,"sense_key":"draw%2:34:01::","synset_id":"puff.v.02"},{"index":35,"sense_key":"draw%2:38:02::","synset_id":"draw.v.12"},{"index":36,"sense_key":"draw%2:30:08::","synset_id":"draw.v.32"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"draw%2:35:06::","call_ids":["senseval2.d000.s047.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"draw%2:35:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1865,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.017485,"input_uncached_usd":0.009325,"input_cached_usd":0.0,"output_usd":0.00816,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.921503541991115}
{"item_id":"senseval2.d000.s047.t003","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"},{"index":4,"sense_key":"people%1:14:00::","synset_id":"people.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval2.d000.s047.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.00513,"input_uncached_usd":0.00255,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.264490250032395}
{"item_id":"senseval2.d000.s047.t004","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:01::","synset_id":"church.n.04"},{"index":2,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":3,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":4,"sense_key":"church%1:06:00::","synset_id":"church.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"church%1:14:01::","call_ids":["senseval2.d000.s047.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"church%1:14:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018255,"input_uncached_usd":0.002415,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.90240358398296}
{"item_id":"senseval2.d000.s047.t005","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":2,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":3,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":7,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":8,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s047.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":814,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01991,"input_uncached_usd":0.004070000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.331213250057772}
{"item_id":"senseval2.d000.s048.t000","gold_sense_keys":["live%2:42:06::"],"candidates":[{"index":1,"sense_key":"live%2:42:00::","synset_id":"be.v.11"},{"index":2,"sense_key":"live%2:42:07::","synset_id":"live.v.07"},{"index":3,"sense_key":"live%2:42:04::","synset_id":"survive.v.01"},{"index":4,"sense_key":"live%2:42:08::","synset_id":"populate.v.01"},{"index":5,"sense_key":"live%2:42:01::","synset_id":"exist.v.02"},{"index":6,"sense_key":"live%2:31:00::","synset_id":"know.v.05"},{"index":7,"sense_key":"live%2:42:06::","synset_id":"live.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"live%2:42:06::","call_ids":["senseval2.d000.s048.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"live%2:42:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0193,"input_uncached_usd":0.0034600000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.160157958045602}
{"item_id":"senseval2.d000.s048.t001","gold_sense_keys":["hope%1:12:01::"],"candidates":[{"index":1,"sense_key":"hope%1:12:00::","synset_id":"hope.n.02"},{"index":2,"sense_key":"hope%1:18:01::","synset_id":"hope.n.05"},{"index":3,"sense_key":"hope%1:18:00::","synset_id":"hope.n.04"},{"index":4,"sense_key":"hope%1:09:00::","synset_id":"promise.n.02"},{"index":5,"sense_key":"hope%1:07:00::","synset_id":"hope.n.06"},{"index":6,"sense_key":"hope%1:12:01::","synset_id":"hope.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"hope%1:12:01::","call_ids":["senseval2.d000.s048.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"hope%1:12:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":317,"reasoning_output_tokens":301},"cost":{"currency":"USD","total_usd":0.012334999999999999,"input_uncached_usd":0.0028250000000000003,"input_cached_usd":0.0,"output_usd":0.009510000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.925100791966543}
{"item_id":"senseval2.d000.s048.t002","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"},{"index":4,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s048.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.003865,"input_uncached_usd":0.002395,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.049172875005752}
{"item_id":"senseval2.d000.s048.t003","gold_sense_keys":["draw%2:30:14::"],"candidates":[{"index":1,"sense_key":"draw%2:40:00::","synset_id":"reap.v.02"},{"index":2,"sense_key":"draw%2:37:01::","synset_id":"draw.v.10"},{"index":3,"sense_key":"draw%2:35:06::","synset_id":"attract.v.01"},{"index":4,"sense_key":"draw%2:40:01::","synset_id":"withdraw.v.09"},{"index":5,"sense_key":"draw%2:30:04::","synset_id":"draw.v.35"},{"index":6,"sense_key":"draw%2:30:15::","synset_id":"draw.v.33"},{"index":7,"sense_key":"draw%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"draw%2:32:00::","synset_id":"describe.v.01"},{"index":9,"sense_key":"draw%2:30:05::","synset_id":"disembowel.v.01"},{"index":10,"sense_key":"draw%2:35:03::","synset_id":"pull.v.01"},{"index":11,"sense_key":"draw%2:38:11::","synset_id":"pull.v.09"},{"index":12,"sense_key":"draw%2:36:02::","synset_id":"draw.v.19"},{"index":13,"sense_key":"draw%2:30:08::","synset_id":"draw.v.32"},{"index":14,"sense_key":"draw%2:35:04::","synset_id":"absorb.v.04"},{"index":15,"sense_key":"draw%2:36:00::","synset_id":"draw.v.06"},{"index":16,"sense_key":"draw%2:36:01::","synset_id":"draw.v.15"},{"index":17,"sense_key":"draw%2:38:00::","synset_id":"draw.v.07"},{"index":18,"sense_key":"draw%2:35:09::","synset_id":"pull_back.v.04"},{"index":19,"sense_key":"draw%2:38:01::","synset_id":"draw.v.05"},{"index":20,"sense_key":"draw%2:30:14::","synset_id":"draw.v.16"},{"index":21,"sense_key":"draw%2:35:01::","synset_id":"trace.v.02"},{"index":22,"sense_key":"draw%2:41:13::","synset_id":"draw.v.23"},{"index":23,"sense_key":"draw%2:42:02::","synset_id":"draw.v.22"},{"index":24,"sense_key":"draw%2:30:00::","synset_id":"draw.v.31"},{"index":25,"sense_key":"draw%2:33:01::","synset_id":"draw.v.14"},{"index":26,"sense_key":"draw%2:29:00::","synset_id":"draw.v.36"},{"index":27,"sense_key":"draw%2:35:13::","synset_id":"guide.v.05"},{"index":28,"sense_key":"draw%2:36:07::","synset_id":"draw.v.18"},{"index":29,"sense_key":"draw%2:38:07::","synset_id":"draw.v.17"},{"index":30,"sense_key":"draw%2:35:14::","synset_id":"string.v.01"},{"index":31,"sense_key":"draw%2:34:01::","synset_id":"puff.v.02"},{"index":32,"sense_key":"draw%2:31:13::","synset_id":"draw.v.09"},{"index":33,"sense_key":"draw%2:33:00::","synset_id":"tie.v.02"},{"index":34,"sense_key":"draw%2:38:02::","synset_id":"draw.v.12"},{"index":35,"sense_key":"draw%2:42:00::","synset_id":"draw.v.21"},{"index":36,"sense_key":"draw%2:35:02::","synset_id":"draw.v.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":20,"chosen_sense_key":"draw%2:30:14::","call_ids":["senseval2.d000.s048.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":20,"predicted_sense_key":"draw%2:30:14::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1875,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.025215,"input_uncached_usd":0.009375000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.88185629202053}
{"item_id":"senseval2.d000.s048.t004","gold_sense_keys":["full%3:00:00::"],"candidates":[{"index":1,"sense_key":"full%3:00:00::","synset_id":"full.a.01"},{"index":2,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":3,"sense_key":"full%5:00:00:nourished:00","synset_id":"full.s.03"},{"index":4,"sense_key":"full%3:00:01::","synset_id":"full.a.05"},{"index":5,"sense_key":"full%5:00:02:ample:00","synset_id":"wide.s.04"},{"index":6,"sense_key":"full%5:00:01:whole:00","synset_id":"entire.s.01"},{"index":7,"sense_key":"full%5:00:00:complete:00","synset_id":"full.s.02"},{"index":8,"sense_key":"full%5:00:00:high:02","synset_id":"broad.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"full%3:00:00::","call_ids":["senseval2.d000.s048.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"full%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":667,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019145000000000002,"input_uncached_usd":0.0033350000000000003,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.220748084015213}
{"item_id":"senseval2.d000.s048.t005","gold_sense_keys":["life%1:26:01::"],"candidates":[{"index":1,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":2,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":3,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":7,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":8,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":9,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":10,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":11,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":12,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"},{"index":13,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":14,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"life%1:26:01::","call_ids":["senseval2.d000.s048.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"life%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":908,"cached_input_tokens":0,"output_tokens":423,"reasoning_output_tokens":407},"cost":{"currency":"USD","total_usd":0.017230000000000002,"input_uncached_usd":0.004540000000000001,"input_cached_usd":0.0,"output_usd":0.01269,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.977979541989043}
{"item_id":"senseval2.d000.s049.t000","gold_sense_keys":["sort%1:07:00::"],"candidates":[{"index":1,"sense_key":"sort%1:22:00::","synset_id":"sort.n.04"},{"index":2,"sense_key":"sort%1:07:00::","synset_id":"sort.n.02"},{"index":3,"sense_key":"sort%1:09:00::","synset_id":"kind.n.01"},{"index":4,"sense_key":"sort%1:18:00::","synset_id":"sort.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sort%1:07:00::","call_ids":["senseval2.d000.s049.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sort%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":215,"reasoning_output_tokens":199},"cost":{"currency":"USD","total_usd":0.009125000000000001,"input_uncached_usd":0.0026750000000000003,"input_cached_usd":0.0,"output_usd":0.00645,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.09929841698613}
{"item_id":"senseval2.d000.s049.t001","gold_sense_keys":["parliament%1:14:00::"],"candidates":[{"index":1,"sense_key":"parliament%1:14:00::","synset_id":"parliament.n.01"},{"index":2,"sense_key":"parliament%1:04:00::","synset_id":"fantan.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parliament%1:14:00::","call_ids":["senseval2.d000.s049.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parliament%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":438,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.00429,"input_uncached_usd":0.00219,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.09784708393272}
{"item_id":"senseval2.d000.s049.t002","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:09:00::","synset_id":"group.n.03"},{"index":2,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":3,"sense_key":"group%1:27:00::","synset_id":"group.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"group%1:03:00::","call_ids":["senseval2.d000.s049.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.004665000000000001,"input_uncached_usd":0.0023550000000000003,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.437936250003986}
{"item_id":"senseval2.d000.s049.t003","gold_sense_keys":["aim%2:31:00::","aim%2:31:01::"],"candidates":[{"index":1,"sense_key":"aim%2:31:01::","synset_id":"aim.v.02"},{"index":2,"sense_key":"aim%2:33:10::","synset_id":"target.v.01"},{"index":3,"sense_key":"aim%2:32:09::","synset_id":"drive.v.11"},{"index":4,"sense_key":"aim%2:32:00::","synset_id":"aim.v.06"},{"index":5,"sense_key":"aim%2:31:00::","synset_id":"draw_a_bead_on.v.02"},{"index":6,"sense_key":"aim%2:31:02::","synset_id":"calculate.v.05"},{"index":7,"sense_key":"aim%2:33:00::","synset_id":"aim.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"aim%2:31:01::","call_ids":["senseval2.d000.s049.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"aim%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":104},"cost":{"currency":"USD","total_usd":0.00691,"input_uncached_usd":0.0033100000000000004,"input_cached_usd":0.0,"output_usd":0.0036,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7311227089958265}
{"item_id":"senseval2.d000.s049.t004","gold_sense_keys":["improve%2:30:01::"],"candidates":[{"index":1,"sense_key":"improve%2:30:01::","synset_id":"better.v.02"},{"index":2,"sense_key":"improve%2:30:00::","synset_id":"better.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"improve%2:30:01::","call_ids":["senseval2.d000.s049.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"improve%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.0037,"input_uncached_usd":0.0022,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.661746040917933}
{"item_id":"senseval2.d000.s049.t006","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"},{"index":2,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"},{"index":3,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s049.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.00506,"input_uncached_usd":0.0022400000000000002,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.040586541988887}
{"item_id":"senseval2.d000.s049.t007","gold_sense_keys":["say%2:32:01::"],"candidates":[{"index":1,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":2,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":3,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":4,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":5,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":6,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":7,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":8,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":9,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":10,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":11,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d000.s049.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":837,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020025,"input_uncached_usd":0.004185,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.679782666033134}
{"item_id":"senseval2.d000.s049.t008","gold_sense_keys":["president%1:18:03::"],"candidates":[{"index":1,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":2,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":5,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"},{"index":6,"sense_key":"president%1:18:01::","synset_id":"president.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"president%1:18:03::","call_ids":["senseval2.d000.s049.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"president%1:18:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":593,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.005725,"input_uncached_usd":0.002965,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.534035541000776}
{"item_id":"senseval2.d000.s050.t000","gold_sense_keys":["hope%2:31:00::"],"candidates":[{"index":1,"sense_key":"hope%2:31:00::","synset_id":"hope.v.03"},{"index":2,"sense_key":"hope%2:37:01::","synset_id":"hope.v.01"},{"index":3,"sense_key":"hope%2:37:00::","synset_id":"hope.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hope%2:31:00::","call_ids":["senseval2.d000.s050.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hope%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":94},"cost":{"currency":"USD","total_usd":0.00566,"input_uncached_usd":0.00239,"input_cached_usd":0.0,"output_usd":0.00327,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9335244999965653}
{"item_id":"senseval2.d000.s050.t001","gold_sense_keys":["speak%2:32:01::","speak%2:32:03::"],"candidates":[{"index":1,"sense_key":"speak%2:32:02::","synset_id":"speak.v.03"},{"index":2,"sense_key":"speak%2:32:01::","synset_id":"talk.v.01"},{"index":3,"sense_key":"speak%2:39:00::","synset_id":"speak.v.05"},{"index":4,"sense_key":"speak%2:32:00::","synset_id":"talk.v.02"},{"index":5,"sense_key":"speak%2:32:03::","synset_id":"address.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"speak%2:32:03::","call_ids":["senseval2.d000.s050.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"speak%2:32:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":113},"cost":{"currency":"USD","total_usd":0.0065450000000000005,"input_uncached_usd":0.0026750000000000003,"input_cached_usd":0.0,"output_usd":0.00387,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.252170957974158}
{"item_id":"senseval2.d000.s050.t002","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d000.s050.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.0036950000000000004,"input_uncached_usd":0.002135,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.766136542079039}
{"item_id":"senseval2.d000.s050.t004","gold_sense_keys":["college%1:14:01::"],"candidates":[{"index":1,"sense_key":"college%1:14:01::","synset_id":"college.n.02"},{"index":2,"sense_key":"college%1:14:00::","synset_id":"college.n.01"},{"index":3,"sense_key":"college%1:06:00::","synset_id":"college.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"college%1:14:01::","call_ids":["senseval2.d000.s050.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"college%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":86},"cost":{"currency":"USD","total_usd":0.0052450000000000005,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.538233167026192}
{"item_id":"senseval2.d000.s050.t005","gold_sense_keys":["joy%1:09:00::"],"candidates":[{"index":1,"sense_key":"joy%1:09:00::","synset_id":"joy.n.02"},{"index":2,"sense_key":"joy%1:12:00::","synset_id":"joy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"joy%1:09:00::","call_ids":["senseval2.d000.s050.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"joy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":423,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.004785,"input_uncached_usd":0.002115,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0100613749818876}
{"item_id":"senseval2.d000.s050.t006","gold_sense_keys":["bell_ringing%1:04:00::"],"candidates":[{"index":1,"sense_key":"bell_ringing%1:04:00::","synset_id":"bell_ringing.n.03"},{"index":2,"sense_key":"bell_ringing%1:10:00::","synset_id":"electioneering.n.01"},{"index":3,"sense_key":"bell_ringing%1:11:00::","synset_id":"bell_ringing.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell_ringing%1:04:00::","call_ids":["senseval2.d000.s050.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell_ringing%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.004350000000000001,"input_uncached_usd":0.0022800000000000003,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.74864029197488}
{"item_id":"senseval2.d000.s050.t007","gold_sense_keys":["shortly%4:02:00::"],"candidates":[{"index":1,"sense_key":"shortly%4:02:01::","synset_id":"shortly.r.01"},{"index":2,"sense_key":"shortly%4:02:03::","synset_id":"shortly.r.05"},{"index":3,"sense_key":"shortly%4:02:02::","synset_id":"curtly.r.01"},{"index":4,"sense_key":"shortly%4:02:00::","synset_id":"soon.r.01"},{"index":5,"sense_key":"shortly%4:02:04::","synset_id":"concisely.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"shortly%4:02:00::","call_ids":["senseval2.d000.s050.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"shortly%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.00427,"input_uncached_usd":0.00286,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.316942833014764}
{"item_id":"senseval2.d000.s050.t008","gold_sense_keys":["publish%2:32:00::","publish%2:36:01::"],"candidates":[{"index":1,"sense_key":"publish%2:36:00::","synset_id":"publish.v.03"},{"index":2,"sense_key":"publish%2:36:01::","synset_id":"print.v.01"},{"index":3,"sense_key":"publish%2:32:00::","synset_id":"publish.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"publish%2:32:00::","call_ids":["senseval2.d000.s050.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"publish%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.00515,"input_uncached_usd":0.0024500000000000004,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.26512279198505}
{"item_id":"senseval2.d000.s050.t010","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"},{"index":2,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"},{"index":3,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s050.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.004815,"input_uncached_usd":0.0022050000000000004,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.330247374949977}
{"item_id":"senseval2.d000.s050.t011","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":2,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":3,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":4,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":5,"sense_key":"country%1:15:01::","synset_id":"country.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"country%1:15:00::","call_ids":["senseval2.d000.s050.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"country%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.006065,"input_uncached_usd":0.002975,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6941734589636326}
{"item_id":"senseval2.d000.s050.t012","gold_sense_keys":["entitle%2:32:00::"],"candidates":[{"index":1,"sense_key":"entitle%2:32:00::","synset_id":"entitle.v.02"},{"index":2,"sense_key":"entitle%2:41:01::","synset_id":"ennoble.v.02"},{"index":3,"sense_key":"entitle%2:41:00::","synset_id":"entitle.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"entitle%2:32:00::","call_ids":["senseval2.d000.s050.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"entitle%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.00393,"input_uncached_usd":0.0022500000000000003,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4754827920114622}
{"item_id":"senseval2.d000.s050.t013","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":2,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":5,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":6,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":7,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"},{"index":10,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s050.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.007350000000000001,"input_uncached_usd":0.00387,"input_cached_usd":0.0,"output_usd":0.00348,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.490354249952361}
{"item_id":"senseval2.d000.s050.t014","gold_sense_keys":["care%1:04:00::","care%1:04:01::","care%1:04:02::"],"candidates":[{"index":1,"sense_key":"care%1:12:01::","synset_id":"concern.n.02"},{"index":2,"sense_key":"care%1:04:00::","synset_id":"care.n.06"},{"index":3,"sense_key":"care%1:12:00::","synset_id":"care.n.04"},{"index":4,"sense_key":"care%1:04:02::","synset_id":"care.n.05"},{"index":5,"sense_key":"care%1:09:00::","synset_id":"caution.n.03"},{"index":6,"sense_key":"care%1:04:01::","synset_id":"care.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"care%1:04:02::","call_ids":["senseval2.d000.s050.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"care%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.00559,"input_uncached_usd":0.0030700000000000002,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8130849160952494}
{"item_id":"senseval2.d000.s051.t000","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":3,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":7,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":8,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":9,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":10,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":11,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s051.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":805,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019865,"input_uncached_usd":0.004025,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.158660875051282}
{"item_id":"senseval2.d000.s051.t002","gold_sense_keys":["recognize%2:31:01::"],"candidates":[{"index":1,"sense_key":"recognize%2:41:01::","synset_id":"recognize.v.09"},{"index":2,"sense_key":"recognize%2:32:00::","synset_id":"acknowledge.v.04"},{"index":3,"sense_key":"recognize%2:31:02::","synset_id":"acknowledge.v.06"},{"index":4,"sense_key":"recognize%2:31:01::","synset_id":"recognize.v.02"},{"index":5,"sense_key":"recognize%2:31:00::","synset_id":"recognize.v.04"},{"index":6,"sense_key":"recognize%2:41:00::","synset_id":"accredit.v.01"},{"index":7,"sense_key":"recognize%2:39:01::","synset_id":"spot.v.02"},{"index":8,"sense_key":"recognize%2:42:00::","synset_id":"recognize.v.08"},{"index":9,"sense_key":"recognize%2:32:01::","synset_id":"greet.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"recognize%2:31:01::","call_ids":["senseval2.d000.s051.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"recognize%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.005815000000000001,"input_uncached_usd":0.003625,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.936741750105284}
{"item_id":"senseval2.d000.s051.t003","gold_sense_keys":["no%4:02:01::"],"candidates":[{"index":1,"sense_key":"no%4:02:01::","synset_id":"no.r.02"},{"index":2,"sense_key":"no%4:02:00::","synset_id":"no.r.03"},{"index":3,"sense_key":"no%4:02:02::","synset_id":"no.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"no%4:02:01::","call_ids":["senseval2.d000.s051.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"no%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":2904,"reasoning_output_tokens":2859},"cost":{"currency":"USD","total_usd":0.089315,"input_uncached_usd":0.0021950000000000003,"input_cached_usd":0.0,"output_usd":0.08712,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":54.332212209003046}
{"item_id":"senseval2.d000.s051.t005","gold_sense_keys":["high%3:00:02::"],"candidates":[{"index":1,"sense_key":"high%5:00:00:malodorous:00","synset_id":"gamey.s.02"},{"index":2,"sense_key":"high%5:00:00:intoxicated:00","synset_id":"high.s.04"},{"index":3,"sense_key":"high%3:00:01::","synset_id":"high.a.02"},{"index":4,"sense_key":"high%3:00:03::","synset_id":"high.a.04"},{"index":5,"sense_key":"high%5:00:00:superior:01","synset_id":"eminent.s.01"},{"index":6,"sense_key":"high%5:00:00:elated:00","synset_id":"high.s.02"},{"index":7,"sense_key":"high%3:00:02::","synset_id":"high.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"high%3:00:02::","call_ids":["senseval2.d000.s051.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"high%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01885,"input_uncached_usd":0.00301,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.716961833066307}
{"item_id":"senseval2.d000.s051.t006","gold_sense_keys":["priority%1:26:00::"],"candidates":[{"index":1,"sense_key":"priority%1:07:00::","synset_id":"priority.n.02"},{"index":2,"sense_key":"priority%1:26:00::","synset_id":"precedence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"priority%1:26:00::","call_ids":["senseval2.d000.s051.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"priority%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00387,"input_uncached_usd":0.0020700000000000002,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.585641416022554}
{"item_id":"senseval2.d000.s051.t007","gold_sense_keys":["church%1:14:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":3,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"church%1:14:00::","call_ids":["senseval2.d000.s051.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"church%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":5040,"reasoning_output_tokens":4970},"cost":{"currency":"USD","total_usd":0.15358,"input_uncached_usd":0.00238,"input_cached_usd":0.0,"output_usd":0.1512,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":98.6597379159648}
{"item_id":"senseval2.d000.s051.t008","gold_sense_keys":["life%1:26:01::"],"candidates":[{"index":1,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":2,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":3,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":4,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":5,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":6,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":10,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":11,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":12,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":13,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"},{"index":14,"sense_key":"life%1:09:00::","synset_id":"life.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"life%1:26:01::","call_ids":["senseval2.d000.s051.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"life%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":892,"cached_input_tokens":0,"output_tokens":170,"reasoning_output_tokens":154},"cost":{"currency":"USD","total_usd":0.00956,"input_uncached_usd":0.00446,"input_cached_usd":0.0,"output_usd":0.0051,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6442972919903696}
{"item_id":"senseval2.d000.s051.t009","gold_sense_keys":["experience%1:09:00::"],"candidates":[{"index":1,"sense_key":"experience%1:09:01::","synset_id":"experience.n.01"},{"index":2,"sense_key":"experience%1:11:00::","synset_id":"experience.n.03"},{"index":3,"sense_key":"experience%1:09:00::","synset_id":"experience.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"experience%1:09:00::","call_ids":["senseval2.d000.s051.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"experience%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":243,"reasoning_output_tokens":227},"cost":{"currency":"USD","total_usd":0.009505000000000001,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.0072900000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.77143308299128}
{"item_id":"senseval2.d000.s052.t002","gold_sense_keys":["attack%2:30:00::"],"candidates":[{"index":1,"sense_key":"attack%2:33:00::","synset_id":"attack.v.01"},{"index":2,"sense_key":"attack%2:33:02::","synset_id":"assail.v.01"},{"index":3,"sense_key":"attack%2:33:01::","synset_id":"attack.v.03"},{"index":4,"sense_key":"attack%2:32:00::","synset_id":"attack.v.02"},{"index":5,"sense_key":"attack%2:29:00::","synset_id":"attack.v.06"},{"index":6,"sense_key":"attack%2:30:00::","synset_id":"attack.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"attack%2:30:00::","call_ids":["senseval2.d000.s052.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"attack%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.005035,"input_uncached_usd":0.003115,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6287852500099689}
{"item_id":"senseval2.d000.s052.t004","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":2,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"problem%1:26:00::","call_ids":["senseval2.d000.s052.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"problem%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018065,"input_uncached_usd":0.002225,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.72476754093077}
{"item_id":"senseval2.d000.s052.t006","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s052.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":36,"reasoning_output_tokens":20},"cost":{"currency":"USD","total_usd":0.00339,"input_uncached_usd":0.00231,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5409662500023842}
{"item_id":"senseval2.d000.s053.t000","gold_sense_keys":["survey%1:04:02::"],"candidates":[{"index":1,"sense_key":"survey%1:04:02::","synset_id":"survey.n.01"},{"index":2,"sense_key":"survey%1:10:00::","synset_id":"sketch.n.03"},{"index":3,"sense_key":"survey%1:04:01::","synset_id":"view.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survey%1:04:02::","call_ids":["senseval2.d000.s053.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survey%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":118},"cost":{"currency":"USD","total_usd":0.006185000000000001,"input_uncached_usd":0.0021950000000000003,"input_cached_usd":0.0,"output_usd":0.0039900000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.256036042002961}
{"item_id":"senseval2.d000.s053.t001","gold_sense_keys":["say%2:32:01::","say%2:42:00::"],"candidates":[{"index":1,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":2,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":3,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":4,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":5,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":6,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":7,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":10,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":11,"sense_key":"say%2:32:08::","synset_id":"say.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d000.s053.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":817,"cached_input_tokens":0,"output_tokens":1564,"reasoning_output_tokens":1535},"cost":{"currency":"USD","total_usd":0.051005,"input_uncached_usd":0.004085,"input_cached_usd":0.0,"output_usd":0.04692,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":27.564979290938936}
{"item_id":"senseval2.d000.s053.t003","gold_sense_keys":["bell_ringer%1:18:01::"],"candidates":[{"index":1,"sense_key":"bell_ringer%1:18:01::","synset_id":"toller.n.01"},{"index":2,"sense_key":"bell_ringer%1:04:00::","synset_id":"bell_ringer.n.03"},{"index":3,"sense_key":"bell_ringer%1:18:00::","synset_id":"bell_ringer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell_ringer%1:18:01::","call_ids":["senseval2.d000.s053.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell_ringer%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":458,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.004690000000000001,"input_uncached_usd":0.0022900000000000004,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3452850829344243}
{"item_id":"senseval2.d000.s053.t005","gold_sense_keys":["today%1:28:00::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"today%1:28:00::","call_ids":["senseval2.d000.s053.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"today%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":411,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.0042450000000000005,"input_uncached_usd":0.002055,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.052648334065452}
{"item_id":"senseval2.d000.s053.t006","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":2,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":3,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":4,"sense_key":"only%4:02:05::","synset_id":"only.r.07"},{"index":5,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":6,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":7,"sense_key":"only%4:02:00::","synset_id":"only.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"only%4:02:02::","call_ids":["senseval2.d000.s053.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"only%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":235,"reasoning_output_tokens":219},"cost":{"currency":"USD","total_usd":0.01012,"input_uncached_usd":0.0030700000000000002,"input_cached_usd":0.0,"output_usd":0.00705,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.075661290902644}
{"item_id":"senseval2.d000.s053.t007","gold_sense_keys":["still%4:02:01::"],"candidates":[{"index":1,"sense_key":"still%4:02:02::","synset_id":"still.r.04"},{"index":2,"sense_key":"still%4:02:01::","synset_id":"still.r.01"},{"index":3,"sense_key":"still%4:02:04::","synset_id":"however.r.01"},{"index":4,"sense_key":"still%4:02:00::","synset_id":"even.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"still%4:02:01::","call_ids":["senseval2.d000.s053.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"still%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":550,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00455,"input_uncached_usd":0.0027500000000000003,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.867253875010647}
{"item_id":"senseval2.d000.s054.t001","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":4,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s054.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":38,"reasoning_output_tokens":23},"cost":{"currency":"USD","total_usd":0.0035150000000000003,"input_uncached_usd":0.0023750000000000004,"input_cached_usd":0.0,"output_usd":0.00114,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.1371063330443576}
{"item_id":"senseval2.d000.s054.t002","gold_sense_keys":["always%4:02:04::"],"candidates":[{"index":1,"sense_key":"always%4:02:04::","synset_id":"constantly.r.01"},{"index":2,"sense_key":"always%4:02:00::","synset_id":"always.r.01"},{"index":3,"sense_key":"always%4:02:01::","synset_id":"always.r.05"},{"index":4,"sense_key":"always%4:02:03::","synset_id":"constantly.r.02"},{"index":5,"sense_key":"always%4:02:02::","synset_id":"always.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"always%4:02:04::","call_ids":["senseval2.d000.s054.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"always%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":130},"cost":{"currency":"USD","total_usd":0.007155,"input_uncached_usd":0.002775,"input_cached_usd":0.0,"output_usd":0.00438,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.610056250006892}
{"item_id":"senseval2.d000.s054.t003","gold_sense_keys":["live%2:42:08::"],"candidates":[{"index":1,"sense_key":"live%2:31:00::","synset_id":"know.v.05"},{"index":2,"sense_key":"live%2:42:04::","synset_id":"survive.v.01"},{"index":3,"sense_key":"live%2:42:08::","synset_id":"populate.v.01"},{"index":4,"sense_key":"live%2:42:07::","synset_id":"live.v.07"},{"index":5,"sense_key":"live%2:42:01::","synset_id":"exist.v.02"},{"index":6,"sense_key":"live%2:42:00::","synset_id":"be.v.11"},{"index":7,"sense_key":"live%2:42:06::","synset_id":"live.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"live%2:42:08::","call_ids":["senseval2.d000.s054.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"live%2:42:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.005900000000000001,"input_uncached_usd":0.0034400000000000003,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.071176083991304}
{"item_id":"senseval2.d000.s054.t004","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":2,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":3,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":4,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":5,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":6,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":9,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"},{"index":10,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s054.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.005775000000000001,"input_uncached_usd":0.0038250000000000003,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0390104580437765}
{"item_id":"senseval2.d000.s054.t005","gold_sense_keys":["need%2:34:00::"],"candidates":[{"index":1,"sense_key":"need%2:34:01::","synset_id":"need.v.03"},{"index":2,"sense_key":"need%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"need%2:42:00::","synset_id":"necessitate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"need%2:34:00::","call_ids":["senseval2.d000.s054.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"need%2:34:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":463,"cached_input_tokens":0,"output_tokens":3326,"reasoning_output_tokens":3275},"cost":{"currency":"USD","total_usd":0.102095,"input_uncached_usd":0.0023150000000000002,"input_cached_usd":0.0,"output_usd":0.09978000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":61.862611458986066}
{"item_id":"senseval2.d000.s054.t006","gold_sense_keys":["ring%2:39:01::"],"candidates":[{"index":1,"sense_key":"ring%2:35:03::","synset_id":"surround.v.01"},{"index":2,"sense_key":"ring%2:39:00::","synset_id":"ring.v.01"},{"index":3,"sense_key":"ring%2:39:01::","synset_id":"ring.v.03"},{"index":4,"sense_key":"ring%2:39:04::","synset_id":"resound.v.01"},{"index":5,"sense_key":"ring%2:32:00::","synset_id":"call.v.03"},{"index":6,"sense_key":"ring%2:35:00::","synset_id":"ring.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ring%2:39:01::","call_ids":["senseval2.d000.s054.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ring%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.004745,"input_uncached_usd":0.003095,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.656550999963656}
{"item_id":"senseval2.d000.s054.t007","gold_sense_keys":["small%3:00:00::"],"candidates":[{"index":1,"sense_key":"small%5:00:00:young:00","synset_id":"little.s.01"},{"index":2,"sense_key":"small%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":3,"sense_key":"small%5:00:00:fine:00","synset_id":"small.s.07"},{"index":4,"sense_key":"small%5:00:00:moderate:00","synset_id":"modest.s.01"},{"index":5,"sense_key":"small%5:00:00:soft:04","synset_id":"little.s.03"},{"index":6,"sense_key":"small%5:00:00:little:03","synset_id":"small.s.03"},{"index":7,"sense_key":"small%5:00:00:decreased:00","synset_id":"belittled.s.01"},{"index":8,"sense_key":"small%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":9,"sense_key":"small%5:00:00:limited:00","synset_id":"minor.s.03"},{"index":10,"sense_key":"small%3:00:00::","synset_id":"small.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"small%3:00:00::","call_ids":["senseval2.d000.s054.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"small%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":1183,"reasoning_output_tokens":1155},"cost":{"currency":"USD","total_usd":0.03936,"input_uncached_usd":0.00387,"input_cached_usd":0.0,"output_usd":0.03549,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":23.871557792066596}
{"item_id":"senseval2.d000.s055.t000","gold_sense_keys":["council%1:14:01::"],"candidates":[{"index":1,"sense_key":"council%1:14:00::","synset_id":"council.n.03"},{"index":2,"sense_key":"council%1:14:02::","synset_id":"council.n.02"},{"index":3,"sense_key":"council%1:14:01::","synset_id":"council.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"council%1:14:01::","call_ids":["senseval2.d000.s055.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"council%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.004675,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6258337079780176}
{"item_id":"senseval2.d000.s055.t001","gold_sense_keys":["program%1:09:01::"],"candidates":[{"index":1,"sense_key":"program%1:10:00::","synset_id":"program.n.05"},{"index":2,"sense_key":"program%1:10:01::","synset_id":"broadcast.n.02"},{"index":3,"sense_key":"program%1:10:04::","synset_id":"platform.n.02"},{"index":4,"sense_key":"program%1:10:05::","synset_id":"course_of_study.n.01"},{"index":5,"sense_key":"program%1:09:01::","synset_id":"program.n.02"},{"index":6,"sense_key":"program%1:10:02::","synset_id":"program.n.07"},{"index":7,"sense_key":"program%1:09:00::","synset_id":"plan.n.01"},{"index":8,"sense_key":"program%1:04:00::","synset_id":"program.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"program%1:09:01::","call_ids":["senseval2.d000.s055.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"program%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019445,"input_uncached_usd":0.003605,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.72289408405777}
{"item_id":"senseval2.d000.s055.t002","gold_sense_keys":["attract%2:35:00::"],"candidates":[{"index":1,"sense_key":"attract%2:35:00::","synset_id":"attract.v.01"},{"index":2,"sense_key":"attract%2:35:01::","synset_id":"attract.v.03"},{"index":3,"sense_key":"attract%2:37:00::","synset_id":"attract.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attract%2:35:00::","call_ids":["senseval2.d000.s055.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attract%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":363,"reasoning_output_tokens":347},"cost":{"currency":"USD","total_usd":0.013330000000000002,"input_uncached_usd":0.0024400000000000003,"input_cached_usd":0.0,"output_usd":0.01089,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.771209208061919}
{"item_id":"senseval2.d000.s055.t003","gold_sense_keys":["train%2:31:01::","train%2:32:00::","train%2:41:00::"],"candidates":[{"index":1,"sense_key":"train%2:41:00::","synset_id":"prepare.v.05"},{"index":2,"sense_key":"train%2:41:02::","synset_id":"discipline.v.01"},{"index":3,"sense_key":"train%2:31:01::","synset_id":"train.v.01"},{"index":4,"sense_key":"train%2:31:00::","synset_id":"train.v.02"},{"index":5,"sense_key":"train%2:32:00::","synset_id":"coach.v.01"},{"index":6,"sense_key":"train%2:29:00::","synset_id":"train.v.08"},{"index":7,"sense_key":"train%2:41:03::","synset_id":"train.v.09"},{"index":8,"sense_key":"train%2:35:04::","synset_id":"trail.v.05"},{"index":9,"sense_key":"train%2:41:01::","synset_id":"educate.v.03"},{"index":10,"sense_key":"train%2:38:00::","synset_id":"train.v.10"},{"index":11,"sense_key":"train%2:33:00::","synset_id":"aim.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"train%2:31:01::","call_ids":["senseval2.d000.s055.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"train%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":865,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020165000000000002,"input_uncached_usd":0.004325000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.440220749936998}
{"item_id":"senseval2.d000.s055.t004","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"},{"index":4,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s055.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.0038650000000000004,"input_uncached_usd":0.0023350000000000003,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.739046458969824}
{"item_id":"senseval2.d000.s055.t005","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":2,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":3,"sense_key":"only%4:02:05::","synset_id":"only.r.07"},{"index":4,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":5,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":6,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":7,"sense_key":"only%4:02:06::","synset_id":"only.r.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"only%4:02:02::","call_ids":["senseval2.d000.s055.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"only%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.006285000000000001,"input_uncached_usd":0.0030450000000000004,"input_cached_usd":0.0,"output_usd":0.0032400000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3493700419785455}
{"item_id":"senseval2.d000.s055.t008","gold_sense_keys":["say%2:32:01::","say%2:32:13::"],"candidates":[{"index":1,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":2,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":3,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":4,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":5,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":8,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":9,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":10,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":11,"sense_key":"say%2:42:00::","synset_id":"read.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d000.s055.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":812,"cached_input_tokens":0,"output_tokens":2427,"reasoning_output_tokens":2387},"cost":{"currency":"USD","total_usd":0.07687000000000001,"input_uncached_usd":0.00406,"input_cached_usd":0.0,"output_usd":0.07281,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":47.719076749985106}
{"item_id":"senseval2.d000.s056.t000","gold_sense_keys":["right%4:02:04::"],"candidates":[{"index":1,"sense_key":"right%4:02:09::","synset_id":"right.r.03"},{"index":2,"sense_key":"right%4:02:06::","synset_id":"mighty.r.01"},{"index":3,"sense_key":"right%4:02:00::","synset_id":"right.r.06"},{"index":4,"sense_key":"right%4:02:08::","synset_id":"right.r.02"},{"index":5,"sense_key":"right%4:02:04::","synset_id":"right.r.01"},{"index":6,"sense_key":"right%4:02:07::","synset_id":"right.r.07"},{"index":7,"sense_key":"right%4:02:01::","synset_id":"properly.r.01"},{"index":8,"sense_key":"right%4:02:03::","synset_id":"right.r.04"},{"index":9,"sense_key":"right%4:02:02::","synset_id":"correctly.r.01"},{"index":10,"sense_key":"right%4:02:05::","synset_id":"justly.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"right%4:02:08::","call_ids":["senseval2.d000.s056.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"right%4:02:08::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":696,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01932,"input_uncached_usd":0.0034800000000000005,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":20.714334500022233}
{"item_id":"senseval2.d000.s056.t001","gold_sense_keys":["now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":2,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":3,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":4,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":5,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":6,"sense_key":"now%4:02:06::","synset_id":"now.r.07"},{"index":7,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"now%4:02:04::","call_ids":["senseval2.d000.s056.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"now%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":237,"reasoning_output_tokens":222},"cost":{"currency":"USD","total_usd":0.01026,"input_uncached_usd":0.0031500000000000005,"input_cached_usd":0.0,"output_usd":0.00711,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.040744374971837}
{"item_id":"senseval2.d000.s056.t002","gold_sense_keys":["lucky%3:00:00::"],"candidates":[{"index":1,"sense_key":"lucky%3:00:00::","synset_id":"lucky.a.02"},{"index":2,"sense_key":"lucky%5:00:00:propitious:00","synset_id":"golden.s.06"},{"index":3,"sense_key":"lucky%5:00:00:fortunate:00","synset_id":"lucky.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lucky%3:00:00::","call_ids":["senseval2.d000.s056.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lucky%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":191,"reasoning_output_tokens":175},"cost":{"currency":"USD","total_usd":0.007930000000000001,"input_uncached_usd":0.0022,"input_cached_usd":0.0,"output_usd":0.00573,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.922823417000473}
{"item_id":"senseval2.d000.s056.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d000.s056.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00439,"input_uncached_usd":0.0025900000000000003,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.582799332914874}
{"item_id":"senseval2.d000.s056.t005","gold_sense_keys":["new%3:00:00::","new%5:00:00:inexperienced:00","new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":2,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":3,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":4,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":5,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"},{"index":6,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":7,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":8,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":9,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":10,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":11,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"new%5:00:00:inexperienced:00","call_ids":["senseval2.d000.s056.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"new%5:00:00:inexperienced:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":806,"cached_input_tokens":0,"output_tokens":2205,"reasoning_output_tokens":2166},"cost":{"currency":"USD","total_usd":0.07018,"input_uncached_usd":0.004030000000000001,"input_cached_usd":0.0,"output_usd":0.06615,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":53.12696349993348}
{"item_id":"senseval2.d000.s056.t006","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s056.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":42,"reasoning_output_tokens":27},"cost":{"currency":"USD","total_usd":0.003555,"input_uncached_usd":0.002295,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.075147250085138}
{"item_id":"senseval2.d000.s056.t007","gold_sense_keys":["add%2:32:01::"],"candidates":[{"index":1,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":2,"sense_key":"add%2:42:00::","synset_id":"add.v.06"},{"index":3,"sense_key":"add%2:31:00::","synset_id":"add.v.04"},{"index":4,"sense_key":"add%2:32:01::","synset_id":"add.v.02"},{"index":5,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"},{"index":6,"sense_key":"add%2:32:00::","synset_id":"total.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"add%2:32:01::","call_ids":["senseval2.d000.s056.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"add%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":600,"cached_input_tokens":0,"output_tokens":43,"reasoning_output_tokens":28},"cost":{"currency":"USD","total_usd":0.00429,"input_uncached_usd":0.003,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8177193748997524}
{"item_id":"senseval2.d000.s057.t000","gold_sense_keys":["bright%5:00:00:auspicious:00"],"candidates":[{"index":1,"sense_key":"bright%5:00:00:reverberant:00","synset_id":"bright.s.06"},{"index":2,"sense_key":"bright%5:00:00:intelligent:00","synset_id":"bright.s.02"},{"index":3,"sense_key":"bright%5:00:00:happy:00","synset_id":"bright.s.07"},{"index":4,"sense_key":"bright%5:00:00:colorful:00","synset_id":"bright.s.01"},{"index":5,"sense_key":"bright%3:00:00::","synset_id":"bright.a.01"},{"index":6,"sense_key":"bright%5:00:00:glorious:00","synset_id":"bright.s.05"},{"index":7,"sense_key":"bright%3:00:02::","synset_id":"undimmed.a.01"},{"index":8,"sense_key":"bright%5:00:00:auspicious:00","synset_id":"bright.s.08"},{"index":9,"sense_key":"bright%5:00:00:polished:00","synset_id":"bright.s.04"},{"index":10,"sense_key":"bright%5:00:00:light:06","synset_id":"bright.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"bright%5:00:00:auspicious:00","call_ids":["senseval2.d000.s057.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"bright%5:00:00:auspicious:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":783,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.006165,"input_uncached_usd":0.003915,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9400719580007717}
{"item_id":"senseval2.d000.s057.t001","gold_sense_keys":["sign%1:10:05::"],"candidates":[{"index":1,"sense_key":"sign%1:24:00::","synset_id":"polarity.n.02"},{"index":2,"sense_key":"sign%1:26:00::","synset_id":"sign.n.06"},{"index":3,"sense_key":"sign%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":4,"sense_key":"sign%1:10:05::","synset_id":"sign.n.01"},{"index":5,"sense_key":"sign%1:10:03::","synset_id":"signal.n.01"},{"index":6,"sense_key":"sign%1:06:01::","synset_id":"signboard.n.01"},{"index":7,"sense_key":"sign%1:10:04::","synset_id":"sign.n.11"},{"index":8,"sense_key":"sign%1:10:01::","synset_id":"sign.n.09"},{"index":9,"sense_key":"sign%1:10:06::","synset_id":"sign.n.10"},{"index":10,"sense_key":"sign%1:11:00::","synset_id":"augury.n.01"},{"index":11,"sense_key":"sign%1:10:00::","synset_id":"sign.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"sign%1:10:05::","call_ids":["senseval2.d000.s057.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"sign%1:10:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":785,"cached_input_tokens":0,"output_tokens":204,"reasoning_output_tokens":188},"cost":{"currency":"USD","total_usd":0.010045000000000002,"input_uncached_usd":0.0039250000000000005,"input_cached_usd":0.0,"output_usd":0.0061200000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.084144082968123}
{"item_id":"senseval2.d000.s057.t003","gold_sense_keys":["number%1:07:00::"],"candidates":[{"index":1,"sense_key":"number%1:10:05::","synset_id":"number.n.10"},{"index":2,"sense_key":"number%1:07:01::","synset_id":"number.n.09"},{"index":3,"sense_key":"number%1:07:00::","synset_id":"number.n.01"},{"index":4,"sense_key":"number%1:10:04::","synset_id":"act.n.04"},{"index":5,"sense_key":"number%1:10:03::","synset_id":"phone_number.n.01"},{"index":6,"sense_key":"number%1:23:00::","synset_id":"number.n.02"},{"index":7,"sense_key":"number%1:10:02::","synset_id":"number.n.08"},{"index":8,"sense_key":"number%1:10:00::","synset_id":"issue.n.02"},{"index":9,"sense_key":"number%1:14:00::","synset_id":"number.n.07"},{"index":10,"sense_key":"number%1:10:01::","synset_id":"numeral.n.01"},{"index":11,"sense_key":"number%1:06:00::","synset_id":"number.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"number%1:07:00::","call_ids":["senseval2.d000.s057.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"number%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.005595000000000001,"input_uncached_usd":0.003945000000000001,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7449945419793949}
{"item_id":"senseval2.d000.s057.t004","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":3,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s057.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.00491,"input_uncached_usd":0.00251,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9002802500035614}
{"item_id":"senseval2.d000.s057.t005","gold_sense_keys":["enter%2:33:00::","enter%2:38:00::"],"candidates":[{"index":1,"sense_key":"enter%2:38:00::","synset_id":"enter.v.01"},{"index":2,"sense_key":"enter%2:42:00::","synset_id":"figure.v.02"},{"index":3,"sense_key":"enter%2:41:06::","synset_id":"enroll.v.01"},{"index":4,"sense_key":"enter%2:33:00::","synset_id":"enter.v.02"},{"index":5,"sense_key":"enter%2:35:03::","synset_id":"insert.v.01"},{"index":6,"sense_key":"enter%2:32:04::","synset_id":"record.v.01"},{"index":7,"sense_key":"enter%2:30:10::","synset_id":"embark.v.02"},{"index":8,"sense_key":"enter%2:36:00::","synset_id":"enter.v.06"},{"index":9,"sense_key":"enter%2:41:00::","synset_id":"accede.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"enter%2:33:00::","call_ids":["senseval2.d000.s057.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"enter%2:33:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019165,"input_uncached_usd":0.0033250000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.08181808295194}
{"item_id":"senseval2.d000.s057.t006","gold_sense_keys":["once%4:02:01::"],"candidates":[{"index":1,"sense_key":"once%4:02:02::","synset_id":"once.r.02"},{"index":2,"sense_key":"once%4:02:01::","synset_id":"once.r.03"},{"index":3,"sense_key":"once%4:02:00::","synset_id":"once.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"once%4:02:01::","call_ids":["senseval2.d000.s057.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"once%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0037800000000000004,"input_uncached_usd":0.0021000000000000003,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5021987500367686}
{"item_id":"senseval2.d000.s057.t007","gold_sense_keys":["field%1:09:00::","field%1:26:00::"],"candidates":[{"index":1,"sense_key":"field%1:09:01::","synset_id":"field.n.16"},{"index":2,"sense_key":"field%1:14:02::","synset_id":"field.n.13"},{"index":3,"sense_key":"field%1:15:02::","synset_id":"playing_field.n.02"},{"index":4,"sense_key":"field%1:14:03::","synset_id":"field.n.15"},{"index":5,"sense_key":"field%1:15:05::","synset_id":"field.n.14"},{"index":6,"sense_key":"field%1:17:00::","synset_id":"plain.n.01"},{"index":7,"sense_key":"field%1:15:01::","synset_id":"field.n.03"},{"index":8,"sense_key":"field%1:15:03::","synset_id":"field.n.11"},{"index":9,"sense_key":"field%1:06:00::","synset_id":"airfield.n.01"},{"index":10,"sense_key":"field%1:04:00::","synset_id":"field.n.06"},{"index":11,"sense_key":"field%1:19:00::","synset_id":"field.n.05"},{"index":12,"sense_key":"field%1:26:00::","synset_id":"sphere.n.01"},{"index":13,"sense_key":"field%1:15:00::","synset_id":"field.n.01"},{"index":14,"sense_key":"field%1:15:04::","synset_id":"battlefield.n.01"},{"index":15,"sense_key":"field%1:09:00::","synset_id":"discipline.n.01"},{"index":16,"sense_key":"field%1:14:01::","synset_id":"field.n.12"},{"index":17,"sense_key":"field%1:14:00::","synset_id":"field.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"field%1:26:00::","call_ids":["senseval2.d000.s057.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"field%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1077,"cached_input_tokens":0,"output_tokens":155,"reasoning_output_tokens":139},"cost":{"currency":"USD","total_usd":0.010035,"input_uncached_usd":0.005385,"input_cached_usd":0.0,"output_usd":0.0046500000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5316044579958543}
{"item_id":"senseval2.d000.s057.t009","gold_sense_keys":["third%1:23:00::"],"candidates":[{"index":1,"sense_key":"third%1:04:00::","synset_id":"third_base.n.02"},{"index":2,"sense_key":"third%1:06:00::","synset_id":"third_gear.n.01"},{"index":3,"sense_key":"third%1:23:00::","synset_id":"one-third.n.01"},{"index":4,"sense_key":"third%1:06:01::","synset_id":"third_base.n.01"},{"index":5,"sense_key":"third%1:10:00::","synset_id":"third.n.04"},{"index":6,"sense_key":"third%1:24:00::","synset_id":"third.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"third%1:23:00::","call_ids":["senseval2.d000.s057.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"third%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.00455,"input_uncached_usd":0.00299,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.521748333936557}
{"item_id":"senseval2.d000.s057.t010","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":2,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":3,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"},{"index":4,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s057.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00392,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9260055830236524}
{"item_id":"senseval2.d000.s057.t011","gold_sense_keys":["today%1:28:00::"],"candidates":[{"index":1,"sense_key":"today%1:28:01::","synset_id":"today.n.02"},{"index":2,"sense_key":"today%1:28:00::","synset_id":"today.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"today%1:28:00::","call_ids":["senseval2.d000.s057.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"today%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":382,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.004280000000000001,"input_uncached_usd":0.0019100000000000002,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.938721082988195}
{"item_id":"senseval2.d000.s057.t012","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":2,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":3,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"},{"index":4,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s057.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.0043950000000000005,"input_uncached_usd":0.0025050000000000003,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.720405582920648}
{"item_id":"senseval2.d000.s058.t000","gold_sense_keys":["accept%2:40:03::"],"candidates":[{"index":1,"sense_key":"accept%2:40:01::","synset_id":"bear.v.06"},{"index":2,"sense_key":"accept%2:31:01::","synset_id":"accept.v.04"},{"index":3,"sense_key":"accept%2:32:00::","synset_id":"accept.v.03"},{"index":4,"sense_key":"accept%2:40:03::","synset_id":"accept.v.05"},{"index":5,"sense_key":"accept%2:42:00::","synset_id":"accept.v.08"},{"index":6,"sense_key":"accept%2:31:03::","synset_id":"accept.v.07"},{"index":7,"sense_key":"accept%2:31:00::","synset_id":"accept.v.01"},{"index":8,"sense_key":"accept%2:40:07::","synset_id":"accept.v.09"},{"index":9,"sense_key":"accept%2:31:08::","synset_id":"accept.v.11"},{"index":10,"sense_key":"accept%2:40:12::","synset_id":"take.v.20"},{"index":11,"sense_key":"accept%2:40:00::","synset_id":"accept.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"accept%2:40:03::","call_ids":["senseval2.d000.s058.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"accept%2:40:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":179,"reasoning_output_tokens":163},"cost":{"currency":"USD","total_usd":0.00954,"input_uncached_usd":0.00417,"input_cached_usd":0.0,"output_usd":0.00537,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.3964939169818535}
{"item_id":"senseval2.d000.s058.t002","gold_sense_keys":["however%4:02:00::"],"candidates":[{"index":1,"sense_key":"however%4:02:01::","synset_id":"however.r.04"},{"index":2,"sense_key":"however%4:02:02::","synset_id":"however.r.03"},{"index":3,"sense_key":"however%4:02:04::","synset_id":"however.r.02"},{"index":4,"sense_key":"however%4:02:00::","synset_id":"however.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"however%4:02:00::","call_ids":["senseval2.d000.s058.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"however%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":1045,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.034135000000000006,"input_uncached_usd":0.002785,"input_cached_usd":0.0,"output_usd":0.03135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":19.024270916939713}
{"item_id":"senseval2.d000.s059.t000","gold_sense_keys":["bell_ringing%1:04:00::"],"candidates":[{"index":1,"sense_key":"bell_ringing%1:04:00::","synset_id":"bell_ringing.n.03"},{"index":2,"sense_key":"bell_ringing%1:10:00::","synset_id":"electioneering.n.01"},{"index":3,"sense_key":"bell_ringing%1:11:00::","synset_id":"bell_ringing.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell_ringing%1:04:00::","call_ids":["senseval2.d000.s059.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell_ringing%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":93},"cost":{"currency":"USD","total_usd":0.005645000000000001,"input_uncached_usd":0.0023750000000000004,"input_cached_usd":0.0,"output_usd":0.00327,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5900518749840558}
{"item_id":"senseval2.d000.s059.t001","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["senseval2.d000.s059.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.0044800000000000005,"input_uncached_usd":0.0024100000000000002,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.553791541955434}
{"item_id":"senseval2.d000.s059.t002","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":2,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":3,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"country%1:15:00::","call_ids":["senseval2.d000.s059.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"country%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":104},"cost":{"currency":"USD","total_usd":0.00666,"input_uncached_usd":0.0030600000000000002,"input_cached_usd":0.0,"output_usd":0.0036,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.77145241596736}
{"item_id":"senseval2.d000.s059.t003","gold_sense_keys":["found%2:36:00::","found%2:41:00::"],"candidates":[{"index":1,"sense_key":"detect%2:39:00::","synset_id":"detect.v.01"},{"index":2,"sense_key":"determine%2:32:00::","synset_id":"determine.v.01"},{"index":3,"sense_key":"found%2:41:00::","synset_id":"establish.v.01"},{"index":4,"sense_key":"found%2:36:00::","synset_id":"establish.v.02"},{"index":5,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":6,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":7,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":8,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":9,"sense_key":"found%2:31:00::","synset_id":"establish.v.08"},{"index":10,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":11,"sense_key":"line_up%2:40:00::","synset_id":"line_up.v.02"},{"index":12,"sense_key":"recover%2:40:00::","synset_id":"recover.v.01"},{"index":13,"sense_key":"rule%2:32:00::","synset_id":"rule.v.04"},{"index":14,"sense_key":"receive%2:30:01::","synset_id":"receive.v.02"},{"index":15,"sense_key":"find_oneself%2:30:00::","synset_id":"find_oneself.v.01"},{"index":16,"sense_key":"discover%2:36:00::","synset_id":"discover.v.03"},{"index":17,"sense_key":"witness%2:39:01::","synset_id":"witness.v.02"},{"index":18,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":19,"sense_key":"discover%2:31:00::","synset_id":"discover.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"found%2:41:00::","call_ids":["senseval2.d000.s059.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"found%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1325,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.022435000000000004,"input_uncached_usd":0.006625000000000001,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.194455833989196}
{"item_id":"senseval2.d000.s059.t004","gold_sense_keys":["remain%2:30:00::"],"candidates":[{"index":1,"sense_key":"remain%2:42:03::","synset_id":"stay.v.04"},{"index":2,"sense_key":"remain%2:42:01::","synset_id":"persist.v.03"},{"index":3,"sense_key":"remain%2:42:05::","synset_id":"remain.v.03"},{"index":4,"sense_key":"remain%2:30:00::","synset_id":"stay.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"remain%2:30:00::","call_ids":["senseval2.d000.s059.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"remain%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.00466,"input_uncached_usd":0.00289,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7330746250227094}
{"item_id":"senseval2.d000.s059.t005","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":2,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":3,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"},{"index":4,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fact%1:09:01::","call_ids":["senseval2.d000.s059.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fact%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018500000000000003,"input_uncached_usd":0.00269,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.071189583977684}
{"item_id":"senseval2.d000.s059.t006","gold_sense_keys":["particularly%4:02:00::"],"candidates":[{"index":1,"sense_key":"particularly%4:02:01::","synset_id":"peculiarly.r.01"},{"index":2,"sense_key":"particularly%4:02:00::","synset_id":"particularly.r.01"},{"index":3,"sense_key":"particularly%4:02:02::","synset_id":"particularly.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"particularly%4:02:00::","call_ids":["senseval2.d000.s059.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"particularly%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.004635,"input_uncached_usd":0.002565,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.146313957986422}
{"item_id":"senseval2.d000.s059.t008","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s059.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":562,"cached_input_tokens":0,"output_tokens":206,"reasoning_output_tokens":191},"cost":{"currency":"USD","total_usd":0.00899,"input_uncached_usd":0.0028100000000000004,"input_cached_usd":0.0,"output_usd":0.00618,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.077263875049539}
{"item_id":"senseval2.d000.s059.t009","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:09:00::","synset_id":"group.n.03"},{"index":2,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":3,"sense_key":"group%1:27:00::","synset_id":"group.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"group%1:03:00::","call_ids":["senseval2.d000.s059.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":31,"reasoning_output_tokens":16},"cost":{"currency":"USD","total_usd":0.00334,"input_uncached_usd":0.0024100000000000002,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4228226250270382}
{"item_id":"senseval2.d000.s059.t011","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"},{"index":4,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s059.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.004070000000000001,"input_uncached_usd":0.00251,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3146110409870744}
{"item_id":"senseval2.d000.s059.t013","gold_sense_keys":["most%4:02:00::"],"candidates":[{"index":1,"sense_key":"most%4:02:00::","synset_id":"most.r.01"},{"index":2,"sense_key":"most%4:02:02::","synset_id":"about.r.07"},{"index":3,"sense_key":"most%4:02:01::","synset_id":"most.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"most%4:02:00::","call_ids":["senseval2.d000.s059.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"most%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.00459,"input_uncached_usd":0.00249,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.371298582991585}
{"item_id":"senseval2.d000.s059.t015","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:14:01::","synset_id":"church.n.04"},{"index":4,"sense_key":"church%1:14:00::","synset_id":"church.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s059.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.00498,"input_uncached_usd":0.0025800000000000003,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.449001875007525}
{"item_id":"senseval2.d000.s060.t001","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":2,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":3,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"},{"index":4,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s060.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.0060149999999999995,"input_uncached_usd":0.002925,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0282543329522014}
{"item_id":"senseval2.d000.s060.t002","gold_sense_keys":["file%2:32:02::"],"candidates":[{"index":1,"sense_key":"file%2:32:00::","synset_id":"file.v.05"},{"index":2,"sense_key":"file%2:32:01::","synset_id":"charge.v.06"},{"index":3,"sense_key":"file%2:35:00::","synset_id":"file.v.02"},{"index":4,"sense_key":"file%2:38:00::","synset_id":"file.v.03"},{"index":5,"sense_key":"file%2:32:02::","synset_id":"file.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"file%2:32:02::","call_ids":["senseval2.d000.s060.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"file%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.00623,"input_uncached_usd":0.0027800000000000004,"input_cached_usd":0.0,"output_usd":0.00345,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.533459916943684}
{"item_id":"senseval2.d000.s060.t004","gold_sense_keys":["suit%1:04:00::"],"candidates":[{"index":1,"sense_key":"suit%1:10:01::","synset_id":"courtship.n.01"},{"index":2,"sense_key":"suit%1:06:00::","synset_id":"suit.n.01"},{"index":3,"sense_key":"suit%1:18:00::","synset_id":"suit.n.03"},{"index":4,"sense_key":"suit%1:06:01::","synset_id":"suit.n.06"},{"index":5,"sense_key":"suit%1:10:00::","synset_id":"suit.n.05"},{"index":6,"sense_key":"suit%1:04:00::","synset_id":"lawsuit.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"suit%1:04:00::","call_ids":["senseval2.d000.s060.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"suit%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":682,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.005180000000000001,"input_uncached_usd":0.0034100000000000003,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7143373750150204}
{"item_id":"senseval2.d000.s060.t005","gold_sense_keys":["extent%1:26:00::"],"candidates":[{"index":1,"sense_key":"extent%1:07:00::","synset_id":"extent.n.02"},{"index":2,"sense_key":"extent%1:26:00::","synset_id":"extent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"extent%1:26:00::","call_ids":["senseval2.d000.s060.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"extent%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004355,"input_uncached_usd":0.0023450000000000003,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.174211833975278}
{"item_id":"senseval2.d000.s060.t006","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"},{"index":3,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"problem%1:26:00::","call_ids":["senseval2.d000.s060.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"problem%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":219,"reasoning_output_tokens":203},"cost":{"currency":"USD","total_usd":0.00911,"input_uncached_usd":0.00254,"input_cached_usd":0.0,"output_usd":0.00657,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.5367692920845}
{"item_id":"senseval2.d000.s060.t007","gold_sense_keys":["surface%2:30:00::"],"candidates":[{"index":1,"sense_key":"surface%2:35:00::","synset_id":"coat.v.01"},{"index":2,"sense_key":"surface%2:30:00::","synset_id":"come_on.v.01"},{"index":3,"sense_key":"surface%2:38:00::","synset_id":"surface.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"surface%2:30:00::","call_ids":["senseval2.d000.s060.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"surface%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.004285000000000001,"input_uncached_usd":0.002605,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6962615830125287}
{"item_id":"senseval2.d000.s060.t008","gold_sense_keys":["summer%1:28:00::"],"candidates":[{"index":1,"sense_key":"summer%1:28:01::","synset_id":"summer.n.02"},{"index":2,"sense_key":"summer%1:28:00::","synset_id":"summer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"summer%1:28:00::","call_ids":["senseval2.d000.s060.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"summer%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.004115000000000001,"input_uncached_usd":0.002405,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8165480000898242}
{"item_id":"senseval2.d000.s060.t009","gold_sense_keys":["series%1:14:00::"],"candidates":[{"index":1,"sense_key":"series%1:14:00::","synset_id":"series.n.01"},{"index":2,"sense_key":"series%1:09:00::","synset_id":"series.n.07"},{"index":3,"sense_key":"series%1:24:00::","synset_id":"series.n.05"},{"index":4,"sense_key":"series%1:14:01::","synset_id":"series.n.06"},{"index":5,"sense_key":"series%1:11:00::","synset_id":"series.n.04"},{"index":6,"sense_key":"series%1:10:01::","synset_id":"serial.n.01"},{"index":7,"sense_key":"series%1:10:00::","synset_id":"series.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"series%1:14:00::","call_ids":["senseval2.d000.s060.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"series%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":678,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.005220000000000001,"input_uncached_usd":0.0033900000000000002,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.733135791029781}
{"item_id":"senseval2.d000.s060.t010","gold_sense_keys":["letter%1:10:00::"],"candidates":[{"index":1,"sense_key":"letter%1:10:02::","synset_id":"letter.n.05"},{"index":2,"sense_key":"letter%1:10:03::","synset_id":"letter.n.04"},{"index":3,"sense_key":"letter%1:10:00::","synset_id":"letter.n.01"},{"index":4,"sense_key":"letter%1:18:00::","synset_id":"letter.n.03"},{"index":5,"sense_key":"letter%1:10:01::","synset_id":"letter.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"letter%1:10:00::","call_ids":["senseval2.d000.s060.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"letter%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.00446,"input_uncached_usd":0.0029600000000000004,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.529388791997917}
{"item_id":"senseval2.d000.s060.t012","gold_sense_keys":["newspaper%1:10:00::"],"candidates":[{"index":1,"sense_key":"newspaper%1:27:00::","synset_id":"newspaper.n.04"},{"index":2,"sense_key":"newspaper%1:14:00::","synset_id":"newspaper.n.02"},{"index":3,"sense_key":"newspaper%1:10:00::","synset_id":"newspaper.n.01"},{"index":4,"sense_key":"newspaper%1:06:00::","synset_id":"newspaper.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"newspaper%1:10:00::","call_ids":["senseval2.d000.s060.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"newspaper%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.00457,"input_uncached_usd":0.0028000000000000004,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.238417666987516}
{"item_id":"senseval2.d000.s060.t013","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s060.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.0046300000000000004,"input_uncached_usd":0.0026200000000000004,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3724169589113444}
{"item_id":"senseval2.d000.s061.t000","gold_sense_keys":["writer%1:18:01::"],"candidates":[{"index":1,"sense_key":"writer%1:18:00::","synset_id":"writer.n.01"},{"index":2,"sense_key":"writer%1:18:01::","synset_id":"writer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"writer%1:18:01::","call_ids":["senseval2.d000.s061.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"writer%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.004655,"input_uncached_usd":0.002435,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.872657582978718}
{"item_id":"senseval2.d000.s061.t001","gold_sense_keys":["sign%2:32:02::"],"candidates":[{"index":1,"sense_key":"sign%2:32:00::","synset_id":"sign.v.03"},{"index":2,"sense_key":"sign%2:32:02::","synset_id":"sign.v.01"},{"index":3,"sense_key":"sign%2:35:02::","synset_id":"sign.v.06"},{"index":4,"sense_key":"sign%2:32:03::","synset_id":"bless.v.03"},{"index":5,"sense_key":"sign%2:41:01::","synset_id":"sign.v.04"},{"index":6,"sense_key":"sign%2:32:04::","synset_id":"sign.v.07"},{"index":7,"sense_key":"sign%2:32:01::","synset_id":"sign.v.05"},{"index":8,"sense_key":"sign%2:41:00::","synset_id":"sign.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sign%2:32:02::","call_ids":["senseval2.d000.s061.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sign%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":794,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.006070000000000001,"input_uncached_usd":0.0039700000000000004,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.099656834034249}
{"item_id":"senseval2.d000.s061.t002","gold_sense_keys":["letter%1:10:00::"],"candidates":[{"index":1,"sense_key":"letter%1:18:00::","synset_id":"letter.n.03"},{"index":2,"sense_key":"letter%1:10:02::","synset_id":"letter.n.05"},{"index":3,"sense_key":"letter%1:10:03::","synset_id":"letter.n.04"},{"index":4,"sense_key":"letter%1:10:01::","synset_id":"letter.n.02"},{"index":5,"sense_key":"letter%1:10:00::","synset_id":"letter.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"letter%1:10:00::","call_ids":["senseval2.d000.s061.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"letter%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.004825,"input_uncached_usd":0.0031450000000000002,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7218475840054452}
{"item_id":"senseval2.d000.s061.t005","gold_sense_keys":["male%1:18:00::"],"candidates":[{"index":1,"sense_key":"male%1:05:00::","synset_id":"male.n.01"},{"index":2,"sense_key":"male%1:18:00::","synset_id":"male.n.02"},{"index":3,"sense_key":"male%1:15:00::","synset_id":"male.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"male%1:18:00::","call_ids":["senseval2.d000.s061.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"male%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.004555,"input_uncached_usd":0.002605,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0004042090149596}
{"item_id":"senseval2.d000.s061.t006","gold_sense_keys":["remark%2:32:01::"],"candidates":[{"index":1,"sense_key":"remark%2:32:00::","synset_id":"note.v.01"},{"index":2,"sense_key":"remark%2:32:01::","synset_id":"comment.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"remark%2:32:01::","call_ids":["senseval2.d000.s061.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"remark%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":131},"cost":{"currency":"USD","total_usd":0.007000000000000001,"input_uncached_usd":0.0025900000000000003,"input_cached_usd":0.0,"output_usd":0.00441,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1987153749214485}
{"item_id":"senseval2.d000.s061.t007","gold_sense_keys":["frequency%1:28:00::"],"candidates":[{"index":1,"sense_key":"frequency%1:24:00::","synset_id":"frequency.n.02"},{"index":2,"sense_key":"frequency%1:28:00::","synset_id":"frequency.n.01"},{"index":3,"sense_key":"frequency%1:23:00::","synset_id":"frequency.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"frequency%1:28:00::","call_ids":["senseval2.d000.s061.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"frequency%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.0048850000000000005,"input_uncached_usd":0.002755,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2230956670828164}
{"item_id":"senseval2.d000.s061.t008","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"},{"index":2,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":3,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":4,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s061.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.007430000000000001,"input_uncached_usd":0.0031100000000000004,"input_cached_usd":0.0,"output_usd":0.00432,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.508945957990363}
{"item_id":"senseval2.d000.s061.t010","gold_sense_keys":["suggest%2:32:00::"],"candidates":[{"index":1,"sense_key":"suggest%2:32:02::","synset_id":"suggest.v.03"},{"index":2,"sense_key":"suggest%2:32:00::","synset_id":"propose.v.01"},{"index":3,"sense_key":"suggest%2:32:03::","synset_id":"indicate.v.05"},{"index":4,"sense_key":"suggest%2:32:01::","synset_id":"suggest.v.05"},{"index":5,"sense_key":"suggest%2:32:04::","synset_id":"hint.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"suggest%2:32:00::","call_ids":["senseval2.d000.s061.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"suggest%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.00523,"input_uncached_usd":0.0030700000000000002,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.077818499994464}
{"item_id":"senseval2.d000.s061.t012","gold_sense_keys":["back%4:02:05::"],"candidates":[{"index":1,"sense_key":"back%4:02:00::","synset_id":"back.r.02"},{"index":2,"sense_key":"back%4:02:05::","synset_id":"back.r.03"},{"index":3,"sense_key":"back%4:02:04::","synset_id":"back.r.01"},{"index":4,"sense_key":"back%4:02:01::","synset_id":"back.r.04"},{"index":5,"sense_key":"back%4:02:06::","synset_id":"back.r.05"},{"index":6,"sense_key":"back%4:02:03::","synset_id":"back.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"back%4:02:05::","call_ids":["senseval2.d000.s061.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"back%4:02:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":650,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":122},"cost":{"currency":"USD","total_usd":0.007390000000000001,"input_uncached_usd":0.0032500000000000003,"input_cached_usd":0.0,"output_usd":0.0041400000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.81766062497627}
{"item_id":"senseval2.d000.s061.t013","gold_sense_keys":["traditional%3:00:00::"],"candidates":[{"index":1,"sense_key":"traditional%3:00:00::","synset_id":"traditional.a.01"},{"index":2,"sense_key":"traditional%5:00:00:orthodox:00","synset_id":"traditional.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"traditional%3:00:00::","call_ids":["senseval2.d000.s061.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"traditional%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":80},"cost":{"currency":"USD","total_usd":0.005335,"input_uncached_usd":0.0024850000000000002,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4047142079798505}
{"item_id":"senseval2.d000.s061.t014","gold_sense_keys":["role%1:04:00::","role%1:04:01::"],"candidates":[{"index":1,"sense_key":"role%1:09:00::","synset_id":"character.n.04"},{"index":2,"sense_key":"role%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"role%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"role%1:04:01::","synset_id":"role.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"role%1:04:01::","call_ids":["senseval2.d000.s061.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"role%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01892,"input_uncached_usd":0.0030800000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.675032958039083}
{"item_id":"senseval2.d000.s061.t015","gold_sense_keys":["make%2:36:07::"],"candidates":[{"index":1,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":2,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":3,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":4,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":5,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":6,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":7,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":8,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":9,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":10,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":11,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":12,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":13,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":14,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":15,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":16,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":17,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":18,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":19,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":20,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":21,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":22,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":23,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":24,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":25,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":26,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":27,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":28,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":29,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":30,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":31,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":32,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":33,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":34,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":35,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":36,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":37,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":38,"sense_key":"make%2:29:01::","synset_id":"make.v.49"},{"index":39,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":40,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":41,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":42,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":43,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":44,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":45,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":46,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":47,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":48,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":49,"sense_key":"make%2:31:12::","synset_id":"make.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":47,"chosen_sense_key":"make%2:36:07::","call_ids":["senseval2.d000.s061.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":47,"predicted_sense_key":"make%2:36:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2381,"cached_input_tokens":0,"output_tokens":173,"reasoning_output_tokens":157},"cost":{"currency":"USD","total_usd":0.017095,"input_uncached_usd":0.011905,"input_cached_usd":0.0,"output_usd":0.00519,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.170943291974254}
{"item_id":"senseval2.d000.s061.t016","gold_sense_keys":["tea%1:13:00::"],"candidates":[{"index":1,"sense_key":"tea%1:20:00::","synset_id":"tea.n.03"},{"index":2,"sense_key":"tea%1:13:02::","synset_id":"tea.n.05"},{"index":3,"sense_key":"tea%1:13:01::","synset_id":"tea.n.02"},{"index":4,"sense_key":"tea%1:13:00::","synset_id":"tea.n.01"},{"index":5,"sense_key":"tea%1:14:00::","synset_id":"tea.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tea%1:13:00::","call_ids":["senseval2.d000.s061.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tea%1:13:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":656,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":88},"cost":{"currency":"USD","total_usd":0.0064,"input_uncached_usd":0.0032800000000000004,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4250696669332683}
{"item_id":"senseval2.d000.s061.t017","gold_sense_keys":["meeting%1:14:00::","meeting%1:14:01::"],"candidates":[{"index":1,"sense_key":"meeting%1:04:02::","synset_id":"merging.n.01"},{"index":2,"sense_key":"meeting%1:14:01::","synset_id":"meeting.n.02"},{"index":3,"sense_key":"meeting%1:15:00::","synset_id":"confluence.n.01"},{"index":4,"sense_key":"meeting%1:14:00::","synset_id":"meeting.n.01"},{"index":5,"sense_key":"meeting%1:04:00::","synset_id":"meeting.n.04"},{"index":6,"sense_key":"meeting%1:11:00::","synset_id":"meeting.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"meeting%1:14:00::","call_ids":["senseval2.d000.s061.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"meeting%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":700,"cached_input_tokens":0,"output_tokens":295,"reasoning_output_tokens":279},"cost":{"currency":"USD","total_usd":0.01235,"input_uncached_usd":0.0035,"input_cached_usd":0.0,"output_usd":0.00885,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.8152289589634165}
{"item_id":"senseval2.d000.s062.t000","gold_sense_keys":["torrent%1:23:00::"],"candidates":[{"index":1,"sense_key":"torrent%1:11:00::","synset_id":"torrent.n.02"},{"index":2,"sense_key":"torrent%1:19:01::","synset_id":"downpour.n.01"},{"index":3,"sense_key":"torrent%1:23:00::","synset_id":"flood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"torrent%1:23:00::","call_ids":["senseval2.d000.s062.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"torrent%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":42,"reasoning_output_tokens":27},"cost":{"currency":"USD","total_usd":0.00404,"input_uncached_usd":0.0027800000000000004,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9161315410165116}
{"item_id":"senseval2.d000.s062.t001","gold_sense_keys":["reply%1:10:01::"],"candidates":[{"index":1,"sense_key":"reply%1:10:00::","synset_id":"reply.n.02"},{"index":2,"sense_key":"reply%1:10:01::","synset_id":"answer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"reply%1:10:01::","call_ids":["senseval2.d000.s062.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"reply%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.0056500000000000005,"input_uncached_usd":0.00265,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6309726670151576}
{"item_id":"senseval2.d000.s062.t002","gold_sense_keys":["follow%2:42:00::","follow%2:42:01::","follow%2:42:05::"],"candidates":[{"index":1,"sense_key":"follow%2:36:01::","synset_id":"play_along.v.02"},{"index":2,"sense_key":"follow%2:31:14::","synset_id":"follow.v.23"},{"index":3,"sense_key":"follow%2:41:00::","synset_id":"comply.v.01"},{"index":4,"sense_key":"follow%2:41:02::","synset_id":"watch.v.02"},{"index":5,"sense_key":"follow%2:42:03::","synset_id":"follow.v.10"},{"index":6,"sense_key":"follow%2:41:09::","synset_id":"be.v.08"},{"index":7,"sense_key":"follow%2:41:11::","synset_id":"follow.v.19"},{"index":8,"sense_key":"follow%2:42:05::","synset_id":"follow.v.03"},{"index":9,"sense_key":"follow%2:30:12::","synset_id":"stick_to.v.02"},{"index":10,"sense_key":"follow%2:41:01::","synset_id":"succeed.v.02"},{"index":11,"sense_key":"follow%2:31:00::","synset_id":"trace.v.01"},{"index":12,"sense_key":"follow%2:30:01::","synset_id":"keep_up.v.04"},{"index":13,"sense_key":"follow%2:38:00::","synset_id":"follow.v.01"},{"index":14,"sense_key":"follow%2:30:00::","synset_id":"follow.v.07"},{"index":15,"sense_key":"follow%2:38:13::","synset_id":"pursue.v.02"},{"index":16,"sense_key":"follow%2:42:02::","synset_id":"follow.v.08"},{"index":17,"sense_key":"follow%2:41:10::","synset_id":"follow.v.18"},{"index":18,"sense_key":"follow%2:42:00::","synset_id":"postdate.v.01"},{"index":19,"sense_key":"follow%2:39:13::","synset_id":"surveil.v.01"},{"index":20,"sense_key":"follow%2:40:00::","synset_id":"adopt.v.01"},{"index":21,"sense_key":"follow%2:36:00::","synset_id":"take_after.v.02"},{"index":22,"sense_key":"follow%2:42:04::","synset_id":"come.v.05"},{"index":23,"sense_key":"follow%2:42:01::","synset_id":"follow.v.06"},{"index":24,"sense_key":"follow%2:38:01::","synset_id":"follow.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":23,"chosen_sense_key":"follow%2:42:01::","call_ids":["senseval2.d000.s062.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":23,"predicted_sense_key":"follow%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1477,"cached_input_tokens":0,"output_tokens":427,"reasoning_output_tokens":411},"cost":{"currency":"USD","total_usd":0.020195,"input_uncached_usd":0.007385000000000001,"input_cached_usd":0.0,"output_usd":0.01281,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.744008166948333}
{"item_id":"senseval2.d000.s062.t003","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":2,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"},{"index":3,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":4,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s062.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00494,"input_uncached_usd":0.0031400000000000004,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8860291249584407}
{"item_id":"senseval2.d000.s062.t004","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":2,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s062.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.0043950000000000005,"input_uncached_usd":0.0028350000000000003,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9076800000621006}
{"item_id":"senseval2.d000.s062.t005","gold_sense_keys":["observe%2:32:00::"],"candidates":[{"index":1,"sense_key":"observe%2:31:00::","synset_id":"observe.v.08"},{"index":2,"sense_key":"observe%2:41:02::","synset_id":"observe.v.06"},{"index":3,"sense_key":"observe%2:41:04::","synset_id":"observe.v.09"},{"index":4,"sense_key":"observe%2:41:00::","synset_id":"watch.v.02"},{"index":5,"sense_key":"observe%2:32:00::","synset_id":"note.v.01"},{"index":6,"sense_key":"observe%2:39:01::","synset_id":"note.v.03"},{"index":7,"sense_key":"observe%2:39:02::","synset_id":"detect.v.01"},{"index":8,"sense_key":"observe%2:39:00::","synset_id":"observe.v.04"},{"index":9,"sense_key":"observe%2:41:01::","synset_id":"respect.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"observe%2:32:00::","call_ids":["senseval2.d000.s062.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"observe%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.006240000000000001,"input_uncached_usd":0.00417,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.991478250012733}
{"item_id":"senseval2.d000.s062.t006","gold_sense_keys":["average%5:00:00:common:01"],"candidates":[{"index":1,"sense_key":"average%5:00:00:common:01","synset_id":"average.s.02"},{"index":2,"sense_key":"average%5:00:03:normal:01","synset_id":"modal.s.01"},{"index":3,"sense_key":"average%5:00:02:normal:01","synset_id":"average.s.01"},{"index":4,"sense_key":"average%5:00:00:moderate:00","synset_id":"average.s.04"},{"index":5,"sense_key":"average%5:00:01:normal:01","synset_id":"median.s.01"},{"index":6,"sense_key":"average%5:00:00:ordinary:00","synset_id":"average.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"average%5:00:00:common:01","call_ids":["senseval2.d000.s062.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"average%5:00:00:common:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":777,"cached_input_tokens":0,"output_tokens":315,"reasoning_output_tokens":299},"cost":{"currency":"USD","total_usd":0.013335,"input_uncached_usd":0.0038850000000000004,"input_cached_usd":0.0,"output_usd":0.00945,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.0600892920047045}
{"item_id":"senseval2.d000.s062.t007","gold_sense_keys":["male%3:00:00::"],"candidates":[{"index":1,"sense_key":"male%5:00:01:male:00","synset_id":"male.s.02"},{"index":2,"sense_key":"male%3:00:00::","synset_id":"male.a.01"},{"index":3,"sense_key":"male%5:00:02:masculine:01","synset_id":"male.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"male%3:00:00::","call_ids":["senseval2.d000.s062.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"male%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018725,"input_uncached_usd":0.0028850000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.361724958987907}
{"item_id":"senseval2.d000.s062.t008","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":2,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"},{"index":3,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":4,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s062.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.004935,"input_uncached_usd":0.0028350000000000003,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3221713750390336}
{"item_id":"senseval2.d000.s062.t013","gold_sense_keys":["badly%4:02:00::","badly%4:02:12::"],"candidates":[{"index":1,"sense_key":"badly%4:02:02::","synset_id":"badly.r.01"},{"index":2,"sense_key":"badly%4:02:12::","synset_id":"badly.r.07"},{"index":3,"sense_key":"badly%4:02:10::","synset_id":"badly.r.08"},{"index":4,"sense_key":"badly%4:02:07::","synset_id":"ill.r.02"},{"index":5,"sense_key":"badly%4:02:06::","synset_id":"badly.r.03"},{"index":6,"sense_key":"badly%4:02:08::","synset_id":"badly.r.10"},{"index":7,"sense_key":"badly%4:02:05::","synset_id":"badly.r.04"},{"index":8,"sense_key":"badly%4:02:00::","synset_id":"ill.r.01"},{"index":9,"sense_key":"badly%4:02:03::","synset_id":"badly.r.06"},{"index":10,"sense_key":"badly%4:02:04::","synset_id":"badly.r.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"badly%4:02:00::","call_ids":["senseval2.d000.s062.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"badly%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":925,"cached_input_tokens":0,"output_tokens":186,"reasoning_output_tokens":170},"cost":{"currency":"USD","total_usd":0.010205000000000002,"input_uncached_usd":0.004625000000000001,"input_cached_usd":0.0,"output_usd":0.00558,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.220382916973904}
{"item_id":"senseval2.d000.s062.t014","gold_sense_keys":["dressed%5:00:00:clothed:00","dressed%5:00:02:clothed:00"],"candidates":[{"index":1,"sense_key":"dressed%5:00:00:treated:02","synset_id":"dressed.s.02"},{"index":2,"sense_key":"dressed%5:00:00:finished:02","synset_id":"dressed.s.03"},{"index":3,"sense_key":"dressed%5:00:02:clothed:00","synset_id":"dressed.s.04"},{"index":4,"sense_key":"dressed%5:00:00:clothed:00","synset_id":"appareled.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"dressed%5:00:00:clothed:00","call_ids":["senseval2.d000.s062.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"dressed%5:00:00:clothed:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":126},"cost":{"currency":"USD","total_usd":0.00733,"input_uncached_usd":0.0030700000000000002,"input_cached_usd":0.0,"output_usd":0.00426,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1638058749958873}
{"item_id":"senseval2.d000.s062.t017","gold_sense_keys":["large%3:00:00::"],"candidates":[{"index":1,"sense_key":"large%5:00:00:conspicuous:00","synset_id":"big.s.04"},{"index":2,"sense_key":"large%5:00:00:pregnant:00","synset_id":"big.s.12"},{"index":3,"sense_key":"large%5:00:00:rhetorical:00","synset_id":"bombastic.s.01"},{"index":4,"sense_key":"large%5:00:00:comprehensive:00","synset_id":"large.s.05"},{"index":5,"sense_key":"large%3:00:00::","synset_id":"large.a.01"},{"index":6,"sense_key":"large%5:00:00:significant:00","synset_id":"large.s.01"},{"index":7,"sense_key":"large%5:00:00:generous:02","synset_id":"big.s.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"large%3:00:00::","call_ids":["senseval2.d000.s062.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"large%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.005595000000000001,"input_uncached_usd":0.0038550000000000004,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.917660999926738}
{"item_id":"senseval2.d000.s062.t019","gold_sense_keys":["unwashed%5:00:00:dirty:01"],"candidates":[{"index":1,"sense_key":"unwashed%5:00:00:lowborn:00","synset_id":"common.s.04"},{"index":2,"sense_key":"unwashed%5:00:00:dirty:01","synset_id":"unwashed.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"unwashed%5:00:00:dirty:01","call_ids":["senseval2.d000.s062.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"unwashed%5:00:00:dirty:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.004875000000000001,"input_uncached_usd":0.0026550000000000002,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.934936625068076}
{"item_id":"senseval2.d000.s063.t000","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"},{"index":4,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s063.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":98},"cost":{"currency":"USD","total_usd":0.006435,"input_uncached_usd":0.0030450000000000004,"input_cached_usd":0.0,"output_usd":0.0033900000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.091049332986586}
{"item_id":"senseval2.d000.s063.t001","gold_sense_keys":["write%2:32:01::"],"candidates":[{"index":1,"sense_key":"write%2:36:04::","synset_id":"compose.v.02"},{"index":2,"sense_key":"write%2:36:03::","synset_id":"spell.v.03"},{"index":3,"sense_key":"write%2:32:00::","synset_id":"write.v.02"},{"index":4,"sense_key":"write%2:32:01::","synset_id":"write.v.05"},{"index":5,"sense_key":"write%2:36:02::","synset_id":"publish.v.03"},{"index":6,"sense_key":"write%2:36:00::","synset_id":"write.v.01"},{"index":7,"sense_key":"write%2:36:01::","synset_id":"write.v.07"},{"index":8,"sense_key":"write%2:36:10::","synset_id":"write.v.10"},{"index":9,"sense_key":"write%2:32:08::","synset_id":"write.v.04"},{"index":10,"sense_key":"write%2:32:03::","synset_id":"write.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"write%2:32:01::","call_ids":["senseval2.d000.s063.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"write%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":814,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01991,"input_uncached_usd":0.004070000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.634622041950934}
{"item_id":"senseval2.d000.s063.t002","gold_sense_keys":["say%2:32:00::"],"candidates":[{"index":1,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":2,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":3,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":6,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":7,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":8,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":9,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":10,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":11,"sense_key":"say%2:32:08::","synset_id":"say.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s063.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":894,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.03585,"input_uncached_usd":0.00447,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":20.349743540980853}
{"item_id":"senseval2.d000.s063.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d000.s063.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":90},"cost":{"currency":"USD","total_usd":0.006185,"input_uncached_usd":0.0030350000000000004,"input_cached_usd":0.0,"output_usd":0.00315,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3119578750338405}
{"item_id":"senseval2.d000.s063.t004","gold_sense_keys":["never%4:02:00::"],"candidates":[{"index":1,"sense_key":"never%4:02:01::","synset_id":"never.r.02"},{"index":2,"sense_key":"never%4:02:00::","synset_id":"never.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"never%4:02:00::","call_ids":["senseval2.d000.s063.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"never%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.0048400000000000006,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9977796250022948}
{"item_id":"senseval2.d000.s063.t005","gold_sense_keys":["know%2:31:04::"],"candidates":[{"index":1,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":2,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":3,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":6,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":7,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":8,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":9,"sense_key":"know%2:31:14::","synset_id":"know.v.11"},{"index":10,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":11,"sense_key":"know%2:31:03::","synset_id":"know.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"know%2:31:04::","call_ids":["senseval2.d000.s063.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"know%2:31:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":926,"cached_input_tokens":0,"output_tokens":797,"reasoning_output_tokens":775},"cost":{"currency":"USD","total_usd":0.028540000000000003,"input_uncached_usd":0.0046300000000000004,"input_cached_usd":0.0,"output_usd":0.02391,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.654115083045326}
{"item_id":"senseval2.d000.s063.t006","gold_sense_keys":["lady%1:18:02::"],"candidates":[{"index":1,"sense_key":"lady%1:18:00::","synset_id":"lady.n.03"},{"index":2,"sense_key":"lady%1:18:02::","synset_id":"lady.n.01"},{"index":3,"sense_key":"lady%1:18:01::","synset_id":"dame.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"lady%1:18:02::","call_ids":["senseval2.d000.s063.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"lady%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":188,"reasoning_output_tokens":172},"cost":{"currency":"USD","total_usd":0.00825,"input_uncached_usd":0.0026100000000000003,"input_cached_usd":0.0,"output_usd":0.00564,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9474249590421095}
{"item_id":"senseval2.d000.s063.t008","gold_sense_keys":["belfry%1:06:00::"],"candidates":[{"index":1,"sense_key":"belfry%1:06:01::","synset_id":"campanile.n.01"},{"index":2,"sense_key":"belfry%1:06:00::","synset_id":"belfry.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"belfry%1:06:00::","call_ids":["senseval2.d000.s063.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"belfry%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":180,"reasoning_output_tokens":164},"cost":{"currency":"USD","total_usd":0.0078000000000000005,"input_uncached_usd":0.0024000000000000002,"input_cached_usd":0.0,"output_usd":0.0054,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4610700419871137}
{"item_id":"senseval2.d000.s064.t000","gold_sense_keys":["see%2:39:00::","see%2:39:02::"],"candidates":[{"index":1,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":2,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":3,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":4,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":5,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"},{"index":6,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":7,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":8,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":9,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":10,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":11,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":12,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":13,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":14,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":15,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":16,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":17,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":20,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":21,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":22,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":23,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":24,"sense_key":"see%2:33:00::","synset_id":"see.v.23"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"see%2:39:02::","call_ids":["senseval2.d000.s064.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"see%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1547,"cached_input_tokens":0,"output_tokens":228,"reasoning_output_tokens":213},"cost":{"currency":"USD","total_usd":0.014575000000000001,"input_uncached_usd":0.0077350000000000006,"input_cached_usd":0.0,"output_usd":0.006840000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.41602654196322}
{"item_id":"senseval2.d000.s064.t001","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":2,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":3,"sense_key":"man%1:05:00::","synset_id":"world.n.08"},{"index":4,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":5,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":6,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":7,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":8,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":9,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":10,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":11,"sense_key":"man%1:18:00::","synset_id":"man.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval2.d000.s064.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":940,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.0069500000000000004,"input_uncached_usd":0.0047,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.942823458928615}
{"item_id":"senseval2.d000.s064.t002","gold_sense_keys":["die%2:30:00::"],"candidates":[{"index":1,"sense_key":"die%2:33:00::","synset_id":"die.v.08"},{"index":2,"sense_key":"die%2:30:05::","synset_id":"die.v.09"},{"index":3,"sense_key":"die%2:30:01::","synset_id":"die.v.10"},{"index":4,"sense_key":"die%2:30:04::","synset_id":"fail.v.04"},{"index":5,"sense_key":"die%2:30:00::","synset_id":"die.v.01"},{"index":6,"sense_key":"die%2:37:01::","synset_id":"die.v.06"},{"index":7,"sense_key":"die%2:35:00::","synset_id":"die.v.07"},{"index":8,"sense_key":"die%2:30:02::","synset_id":"die.v.11"},{"index":9,"sense_key":"die%2:37:00::","synset_id":"die.v.03"},{"index":10,"sense_key":"die%2:37:03::","synset_id":"die.v.05"},{"index":11,"sense_key":"die%2:39:00::","synset_id":"die.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"die%2:30:00::","call_ids":["senseval2.d000.s064.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"die%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":871,"cached_input_tokens":0,"output_tokens":262,"reasoning_output_tokens":246},"cost":{"currency":"USD","total_usd":0.012215,"input_uncached_usd":0.004355,"input_cached_usd":0.0,"output_usd":0.00786,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.869898874894716}
{"item_id":"senseval2.d001.s000.t000","gold_sense_keys":["medical%3:01:00::"],"candidates":[{"index":1,"sense_key":"medical%3:00:00::","synset_id":"medical.a.02"},{"index":2,"sense_key":"medical%3:01:00::","synset_id":"medical.a.01"},{"index":3,"sense_key":"medical%3:01:02::","synset_id":"aesculapian.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medical%3:01:00::","call_ids":["senseval2.d001.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medical%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":310,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0034400000000000003,"input_uncached_usd":0.0015500000000000002,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8981445420067757}
{"item_id":"senseval2.d001.s000.t002","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":2,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":3,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":4,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":5,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":6,"sense_key":"start%2:38:03::","synset_id":"start.v.14"},{"index":7,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":8,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":9,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":10,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":11,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":12,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":13,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":14,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"start%2:30:00::","call_ids":["senseval2.d001.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"start%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":823,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.006635,"input_uncached_usd":0.004115000000000001,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5272246670210734}
{"item_id":"senseval2.d001.s000.t003","gold_sense_keys":["handful%1:23:01::"],"candidates":[{"index":1,"sense_key":"handful%1:23:00::","synset_id":"handful.n.02"},{"index":2,"sense_key":"handful%1:23:01::","synset_id":"handful.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"handful%1:23:01::","call_ids":["senseval2.d001.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"handful%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":266,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.0027400000000000002,"input_uncached_usd":0.00133,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9429714999860153}
{"item_id":"senseval2.d001.s000.t005","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":283,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.003065,"input_uncached_usd":0.001415,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7257192499237135}
{"item_id":"senseval2.d001.s000.t006","gold_sense_keys":["unleash%2:35:02::"],"candidates":[{"index":1,"sense_key":"unleash%2:35:01::","synset_id":"unleash.v.02"},{"index":2,"sense_key":"unleash%2:35:02::","synset_id":"unleash.v.03"},{"index":3,"sense_key":"unleash%2:35:00::","synset_id":"unleash.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"unleash%2:35:02::","call_ids":["senseval2.d001.s000.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"unleash%2:35:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":307,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.004745,"input_uncached_usd":0.0015350000000000001,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2258314170176163}
{"item_id":"senseval2.d001.s000.t007","gold_sense_keys":["chaotic%5:00:00:wild:02"],"candidates":[{"index":1,"sense_key":"chaotic%3:01:00::","synset_id":"chaotic.a.03"},{"index":2,"sense_key":"chaotic%5:00:00:wild:02","synset_id":"chaotic.s.02"},{"index":3,"sense_key":"chaotic%5:00:00:disorganized:00","synset_id":"chaotic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"chaotic%5:00:00:wild:02","call_ids":["senseval2.d001.s000.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"chaotic%5:00:00:wild:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":294,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01731,"input_uncached_usd":0.0014700000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.274693083018064}
{"item_id":"senseval2.d001.s000.t008","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"},{"index":2,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":3,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":4,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":5,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":6,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":7,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"growth%1:26:00::","call_ids":["senseval2.d001.s000.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"growth%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018285,"input_uncached_usd":0.002445,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.77144783292897}
{"item_id":"senseval2.d001.s000.t009","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":5,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s000.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.0036950000000000004,"input_uncached_usd":0.0023450000000000003,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8120050840079784}
{"item_id":"senseval2.d001.s000.t010","gold_sense_keys":["characterize%2:42:01::"],"candidates":[{"index":1,"sense_key":"characterize%2:32:00::","synset_id":"qualify.v.06"},{"index":2,"sense_key":"characterize%2:42:01::","synset_id":"characterize.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"characterize%2:42:01::","call_ids":["senseval2.d001.s000.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"characterize%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":298,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.0030500000000000006,"input_uncached_usd":0.0014900000000000002,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7328714169561863}
{"item_id":"senseval2.d001.s000.t011","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":2,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":3,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":4,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":5,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s000.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":43,"reasoning_output_tokens":27},"cost":{"currency":"USD","total_usd":0.0032700000000000003,"input_uncached_usd":0.00198,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.956374750006944}
{"item_id":"senseval2.d001.s001.t001","gold_sense_keys":["say%2:32:01::","say%2:32:13::"],"candidates":[{"index":1,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":2,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":3,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":4,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":5,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":6,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":9,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":10,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":11,"sense_key":"say%2:32:15::","synset_id":"say.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d001.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019345,"input_uncached_usd":0.0035050000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.185711874975823}
{"item_id":"senseval2.d001.s001.t002","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"},{"index":2,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"discovery%1:04:00::","call_ids":["senseval2.d001.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"discovery%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":348,"cached_input_tokens":0,"output_tokens":324,"reasoning_output_tokens":308},"cost":{"currency":"USD","total_usd":0.01146,"input_uncached_usd":0.0017400000000000002,"input_cached_usd":0.0,"output_usd":0.00972,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.659542458015494}
{"item_id":"senseval2.d001.s001.t004","gold_sense_keys":["recent%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"recent%5:00:00:new:00","synset_id":"recent.s.01"},{"index":2,"sense_key":"recent%5:00:00:past:00","synset_id":"late.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"recent%5:00:00:past:00","call_ids":["senseval2.d001.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"recent%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":294,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.0032400000000000003,"input_uncached_usd":0.0014700000000000002,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6998299170518294}
{"item_id":"senseval2.d001.s001.t005","gold_sense_keys":["month%1:28:01::"],"candidates":[{"index":1,"sense_key":"month%1:28:00::","synset_id":"month.n.02"},{"index":2,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"month%1:28:00::","call_ids":["senseval2.d001.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"month%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":294,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01731,"input_uncached_usd":0.0014700000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.479760541929863}
{"item_id":"senseval2.d001.s001.t007","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":2,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":3,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":4,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":5,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":6,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":7,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":8,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"},{"index":9,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":10,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":11,"sense_key":"new%3:00:09::","synset_id":"new.a.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"new%5:00:00:original:00","call_ids":["senseval2.d001.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"new%5:00:00:original:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01935,"input_uncached_usd":0.00351,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":16.7739315420622}
{"item_id":"senseval2.d001.s001.t009","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":2,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":3,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":33,"reasoning_output_tokens":18},"cost":{"currency":"USD","total_usd":0.003085,"input_uncached_usd":0.002095,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.331434917054139}
{"item_id":"senseval2.d001.s001.t010","gold_sense_keys":["develop%2:30:00::"],"candidates":[{"index":1,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":2,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":3,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":4,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":5,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":6,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":7,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":8,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":9,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":10,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":11,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":14,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":15,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":16,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":19,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":20,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":21,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"develop%2:30:00::","call_ids":["senseval2.d001.s001.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"develop%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1159,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.021635,"input_uncached_usd":0.005795000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.117223917040974}
{"item_id":"senseval2.d001.s002.t001","gold_sense_keys":["understanding%1:09:01::"],"candidates":[{"index":1,"sense_key":"understanding%1:10:00::","synset_id":"agreement.n.01"},{"index":2,"sense_key":"understanding%1:09:03::","synset_id":"sympathy.n.01"},{"index":3,"sense_key":"understanding%1:09:01::","synset_id":"understanding.n.01"},{"index":4,"sense_key":"understanding%1:09:00::","synset_id":"reason.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"understanding%1:09:01::","call_ids":["senseval2.d001.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"understanding%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.00444,"input_uncached_usd":0.00222,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7653372909408063}
{"item_id":"senseval2.d001.s002.t003","gold_sense_keys":["expect%2:31:00::"],"candidates":[{"index":1,"sense_key":"expect%2:29:00::","synset_id":"have_a_bun_in_the_oven.v.01"},{"index":2,"sense_key":"expect%2:29:02::","synset_id":"expect.v.05"},{"index":3,"sense_key":"expect%2:32:00::","synset_id":"ask.v.04"},{"index":4,"sense_key":"expect%2:31:00::","synset_id":"expect.v.01"},{"index":5,"sense_key":"expect%2:31:05::","synset_id":"expect.v.04"},{"index":6,"sense_key":"expect%2:31:01::","synset_id":"expect.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"expect%2:31:00::","call_ids":["senseval2.d001.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"expect%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":112},"cost":{"currency":"USD","total_usd":0.006265,"input_uncached_usd":0.002425,"input_cached_usd":0.0,"output_usd":0.00384,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.855294458102435}
{"item_id":"senseval2.d001.s002.t004","gold_sense_keys":["produce%2:36:03::"],"candidates":[{"index":1,"sense_key":"produce%2:39:00::","synset_id":"produce.v.06"},{"index":2,"sense_key":"produce%2:36:00::","synset_id":"produce.v.02"},{"index":3,"sense_key":"produce%2:39:01::","synset_id":"produce.v.04"},{"index":4,"sense_key":"produce%2:36:02::","synset_id":"produce.v.01"},{"index":5,"sense_key":"produce%2:36:03::","synset_id":"produce.v.03"},{"index":6,"sense_key":"produce%2:29:00::","synset_id":"grow.v.08"},{"index":7,"sense_key":"produce%2:36:05::","synset_id":"grow.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"produce%2:36:03::","call_ids":["senseval2.d001.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"produce%2:36:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":564,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.00534,"input_uncached_usd":0.00282,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.072026832960546}
{"item_id":"senseval2.d001.s002.t005","gold_sense_keys":["array%1:10:00::"],"candidates":[{"index":1,"sense_key":"array%1:06:01::","synset_id":"array.n.03"},{"index":2,"sense_key":"array%1:10:00::","synset_id":"array.n.02"},{"index":3,"sense_key":"array%1:06:00::","synset_id":"array.n.04"},{"index":4,"sense_key":"array%1:14:00::","synset_id":"array.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"array%1:10:00::","call_ids":["senseval2.d001.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"array%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":351,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.004725,"input_uncached_usd":0.001755,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4286465840414166}
{"item_id":"senseval2.d001.s002.t006","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":2,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"},{"index":6,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":7,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":8,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":9,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":10,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":11,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"new%5:00:00:original:00","call_ids":["senseval2.d001.s002.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"new%5:00:00:original:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":708,"cached_input_tokens":0,"output_tokens":1564,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.050460000000000005,"input_uncached_usd":0.00354,"input_cached_usd":0.0,"output_usd":0.04692,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":28.529095834004693}
{"item_id":"senseval2.d001.s002.t007","gold_sense_keys":["strategy%1:09:00::"],"candidates":[{"index":1,"sense_key":"strategy%1:09:01::","synset_id":"strategy.n.02"},{"index":2,"sense_key":"strategy%1:09:00::","synset_id":"scheme.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"strategy%1:09:00::","call_ids":["senseval2.d001.s002.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"strategy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":287,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0033550000000000003,"input_uncached_usd":0.001435,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0379527090117335}
{"item_id":"senseval2.d001.s002.t008","gold_sense_keys":["future%3:00:00::","future%5:00:00:prospective:00"],"candidates":[{"index":1,"sense_key":"future%3:00:00::","synset_id":"future.a.01"},{"index":2,"sense_key":"future%5:00:00:incoming:00","synset_id":"future.s.02"},{"index":3,"sense_key":"future%3:01:00::","synset_id":"future.a.04"},{"index":4,"sense_key":"future%5:00:00:prospective:00","synset_id":"future.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"future%3:00:00::","call_ids":["senseval2.d001.s002.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"future%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":371,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017695,"input_uncached_usd":0.001855,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.590533041046001}
{"item_id":"senseval2.d001.s002.t009","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":3,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":4,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":5,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s002.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":425,"cached_input_tokens":0,"output_tokens":39,"reasoning_output_tokens":24},"cost":{"currency":"USD","total_usd":0.003295,"input_uncached_usd":0.002125,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6894407499348745}
{"item_id":"senseval2.d001.s002.t010","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":2,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":3,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"},{"index":4,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"treatment%1:04:00::","call_ids":["senseval2.d001.s002.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":401,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0036850000000000003,"input_uncached_usd":0.0020050000000000003,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5802614169660956}
{"item_id":"senseval2.d001.s003.t000","gold_sense_keys":["future%1:28:00::"],"candidates":[{"index":1,"sense_key":"future%1:06:00::","synset_id":"future.n.03"},{"index":2,"sense_key":"future%1:28:00::","synset_id":"future.n.01"},{"index":3,"sense_key":"future%1:10:00::","synset_id":"future.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"future%1:28:00::","call_ids":["senseval2.d001.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"future%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":354,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.00321,"input_uncached_usd":0.00177,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9651416250271723}
{"item_id":"senseval2.d001.s004.t002","gold_sense_keys":["develop%2:36:01::"],"candidates":[{"index":1,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":2,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":3,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":4,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":5,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":6,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":7,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":8,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":9,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":10,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":11,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":12,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":13,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":14,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":15,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":16,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":17,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":18,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":19,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"},{"index":20,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":21,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":21,"chosen_sense_key":"develop%2:36:01::","call_ids":["senseval2.d001.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":21,"predicted_sense_key":"develop%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1235,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.009415,"input_uncached_usd":0.006175000000000001,"input_cached_usd":0.0,"output_usd":0.0032400000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0964406660059467}
{"item_id":"senseval2.d001.s004.t004","gold_sense_keys":["base%2:31:00::"],"candidates":[{"index":1,"sense_key":"base%2:34:00::","synset_id":"free-base.v.01"},{"index":2,"sense_key":"base%2:31:00::","synset_id":"establish.v.08"},{"index":3,"sense_key":"base%2:42:04::","synset_id":"base.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"base%2:31:00::","call_ids":["senseval2.d001.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"base%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":408,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.0039,"input_uncached_usd":0.00204,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9769462919794023}
{"item_id":"senseval2.d001.s004.t006","gold_sense_keys":["identify%2:31:01::"],"candidates":[{"index":1,"sense_key":"identify%2:31:00::","synset_id":"identify.v.01"},{"index":2,"sense_key":"identify%2:31:04::","synset_id":"identify.v.03"},{"index":3,"sense_key":"identify%2:31:03::","synset_id":"identify.v.04"},{"index":4,"sense_key":"identify%2:31:02::","synset_id":"identify.v.06"},{"index":5,"sense_key":"identify%2:31:01::","synset_id":"identify.v.05"},{"index":6,"sense_key":"identify%2:32:00::","synset_id":"name.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"identify%2:31:01::","call_ids":["senseval2.d001.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"identify%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":550,"cached_input_tokens":0,"output_tokens":424,"reasoning_output_tokens":408},"cost":{"currency":"USD","total_usd":0.015470000000000001,"input_uncached_usd":0.0027500000000000003,"input_cached_usd":0.0,"output_usd":0.01272,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.428500082925893}
{"item_id":"senseval2.d001.s004.t008","gold_sense_keys":["first%3:00:00::"],"candidates":[{"index":1,"sense_key":"first%5:00:02:opening:00","synset_id":"inaugural.s.01"},{"index":2,"sense_key":"first%3:00:03::","synset_id":"first.a.06"},{"index":3,"sense_key":"first%5:00:01:opening:00","synset_id":"beginning.s.01"},{"index":4,"sense_key":"first%5:00:00:ordinal:00","synset_id":"first.s.01"},{"index":5,"sense_key":"first%5:00:00:best:00","synset_id":"first.s.04"},{"index":6,"sense_key":"first%3:00:00::","synset_id":"first.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"first%3:00:00::","call_ids":["senseval2.d001.s004.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"first%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0186,"input_uncached_usd":0.0027600000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.62744216597639}
{"item_id":"senseval2.d001.s004.t009","gold_sense_keys":["time%1:11:00::"],"candidates":[{"index":1,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":2,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":3,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":4,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":5,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":6,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":7,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":8,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":9,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"},{"index":10,"sense_key":"time%1:11:01::","synset_id":"time.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"time%1:11:00::","call_ids":["senseval2.d001.s004.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"time%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.005645000000000001,"input_uncached_usd":0.0034850000000000003,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8798310830025002}
{"item_id":"senseval2.d001.s004.t011","gold_sense_keys":["otherwise%4:02:01::"],"candidates":[{"index":1,"sense_key":"otherwise%4:02:01::","synset_id":"otherwise.r.01"},{"index":2,"sense_key":"otherwise%4:02:03::","synset_id":"differently.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"otherwise%4:02:01::","call_ids":["senseval2.d001.s004.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"otherwise%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.00398,"input_uncached_usd":0.00185,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5603201671037823}
{"item_id":"senseval2.d001.s004.t012","gold_sense_keys":["healthy%3:00:00::"],"candidates":[{"index":1,"sense_key":"healthy%5:00:00:reasonable:00","synset_id":"healthy.s.03"},{"index":2,"sense_key":"healthy%3:00:00::","synset_id":"healthy.a.01"},{"index":3,"sense_key":"healthy%5:00:00:wholesome:00","synset_id":"healthy.s.02"},{"index":4,"sense_key":"healthy%5:00:00:sound:00","synset_id":"healthy.s.01"},{"index":5,"sense_key":"healthy%5:00:00:considerable:00","synset_id":"goodly.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"healthy%3:00:00::","call_ids":["senseval2.d001.s004.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"healthy%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.0045000000000000005,"input_uncached_usd":0.00267,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.980468083987944}
{"item_id":"senseval2.d001.s004.t013","gold_sense_keys":["individual%1:03:00::"],"candidates":[{"index":1,"sense_key":"individual%1:03:00::","synset_id":"person.n.01"},{"index":2,"sense_key":"individual%1:18:00::","synset_id":"individual.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"individual%1:03:00::","call_ids":["senseval2.d001.s004.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"individual%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":355,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0036650000000000003,"input_uncached_usd":0.001775,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8432376669952646}
{"item_id":"senseval2.d001.s004.t014","gold_sense_keys":["likely%3:00:00::"],"candidates":[{"index":1,"sense_key":"likely%3:00:04::","synset_id":"probable.a.01"},{"index":2,"sense_key":"likely%5:00:00:prospective:00","synset_id":"likely.s.01"},{"index":3,"sense_key":"likely%3:00:00::","synset_id":"likely.a.01"},{"index":4,"sense_key":"likely%5:00:00:credible:00","synset_id":"likely.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"likely%3:00:00::","call_ids":["senseval2.d001.s004.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"likely%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":98},"cost":{"currency":"USD","total_usd":0.005685,"input_uncached_usd":0.002295,"input_cached_usd":0.0,"output_usd":0.0033900000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.103377624996938}
{"item_id":"senseval2.d001.s004.t015","gold_sense_keys":["get%2:29:00::"],"candidates":[{"index":1,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":2,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":3,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":4,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":5,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":6,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":7,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":8,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":9,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":10,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":11,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":12,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":13,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":14,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":15,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":16,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":17,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":18,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":19,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":22,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":23,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":24,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":25,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":26,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":27,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":28,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":29,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":30,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":31,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":32,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":33,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"},{"index":34,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":35,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":36,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":31,"chosen_sense_key":"get%2:29:00::","call_ids":["senseval2.d001.s004.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":31,"predicted_sense_key":"get%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1926,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.011790000000000002,"input_uncached_usd":0.009630000000000001,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3777320000808686}
{"item_id":"senseval2.d001.s004.t016","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s004.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":29,"reasoning_output_tokens":14},"cost":{"currency":"USD","total_usd":0.0033350000000000003,"input_uncached_usd":0.002465,"input_cached_usd":0.0,"output_usd":0.00087,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.350916999974288}
{"item_id":"senseval2.d001.s005.t000","gold_sense_keys":["set%1:14:00::"],"candidates":[{"index":1,"sense_key":"set%1:22:00::","synset_id":"hardening.n.02"},{"index":2,"sense_key":"set%1:18:00::","synset_id":"set.n.10"},{"index":3,"sense_key":"set%1:11:00::","synset_id":"set.n.11"},{"index":4,"sense_key":"set%1:09:01::","synset_id":"bent.n.01"},{"index":5,"sense_key":"set%1:06:00::","synset_id":"stage_set.n.01"},{"index":6,"sense_key":"set%1:06:01::","synset_id":"set.n.13"},{"index":7,"sense_key":"set%1:14:00::","synset_id":"set.n.01"},{"index":8,"sense_key":"set%1:14:02::","synset_id":"set.n.02"},{"index":9,"sense_key":"set%1:04:01::","synset_id":"set.n.07"},{"index":10,"sense_key":"set%1:09:00::","synset_id":"set.n.12"},{"index":11,"sense_key":"set%1:04:00::","synset_id":"set.n.03"},{"index":12,"sense_key":"set%1:14:01::","synset_id":"set.n.05"},{"index":13,"sense_key":"set%1:28:00::","synset_id":"set.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"set%1:14:00::","call_ids":["senseval2.d001.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"set%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":871,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.0065450000000000005,"input_uncached_usd":0.004355,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.051911082933657}
{"item_id":"senseval2.d001.s005.t001","gold_sense_keys":["discovery%1:04:00::","discovery%1:09:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"discovery%1:09:00::","call_ids":["senseval2.d001.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"discovery%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01805,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.018061207956634}
{"item_id":"senseval2.d001.s005.t002","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":2,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":3,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":4,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":5,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":8,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":9,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":10,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"say%2:32:15::","call_ids":["senseval2.d001.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"say%2:32:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":794,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01981,"input_uncached_usd":0.0039700000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.206489208037965}
{"item_id":"senseval2.d001.s005.t005","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":2,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":3,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":4,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":5,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"},{"index":6,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":7,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":8,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":9,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":10,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":11,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":12,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":13,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":16,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"find%2:36:00::","call_ids":["senseval2.d001.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"find%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1105,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.021335,"input_uncached_usd":0.005525,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.486590708955191}
{"item_id":"senseval2.d001.s006.t000","gold_sense_keys":["only%4:02:05::"],"candidates":[{"index":1,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":4,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":5,"sense_key":"only%4:02:05::","synset_id":"only.r.07"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:04::","synset_id":"only.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"only%4:02:05::","call_ids":["senseval2.d001.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"only%4:02:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.005315,"input_uncached_usd":0.002975,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1272799579892308}
{"item_id":"senseval2.d001.s006.t001","gold_sense_keys":["decade%1:28:00::"],"candidates":[{"index":1,"sense_key":"decade%1:23:00::","synset_id":"ten.n.01"},{"index":2,"sense_key":"decade%1:28:00::","synset_id":"decade.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"decade%1:28:00::","call_ids":["senseval2.d001.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"decade%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":397,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.0035450000000000004,"input_uncached_usd":0.0019850000000000002,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6836264579324052}
{"item_id":"senseval2.d001.s006.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":2,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":3,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":4,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.004205,"input_uncached_usd":0.002585,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8383967500412837}
{"item_id":"senseval2.d001.s006.t004","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":2,"sense_key":"know%2:31:14::","synset_id":"know.v.11"},{"index":3,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":4,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":5,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":6,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":7,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":8,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":9,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":10,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":11,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d001.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":832,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":113},"cost":{"currency":"USD","total_usd":0.00803,"input_uncached_usd":0.0041600000000000005,"input_cached_usd":0.0,"output_usd":0.00387,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6984479170059785}
{"item_id":"senseval2.d001.s006.t006","gold_sense_keys":["molecular%3:01:00::"],"candidates":[{"index":1,"sense_key":"molecular%3:00:00::","synset_id":"molecular.a.02"},{"index":2,"sense_key":"molecular%3:01:00::","synset_id":"molecular.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"molecular%3:01:00::","call_ids":["senseval2.d001.s006.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"molecular%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":399,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.0043950000000000005,"input_uncached_usd":0.0019950000000000002,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.260290833073668}
{"item_id":"senseval2.d001.s006.t007","gold_sense_keys":["level%1:09:00::"],"candidates":[{"index":1,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":2,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":3,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":4,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":5,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"},{"index":6,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":7,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":8,"sense_key":"level%1:09:00::","synset_id":"level.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"level%1:09:00::","call_ids":["senseval2.d001.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"level%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019225,"input_uncached_usd":0.0033850000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.640656542032957}
{"item_id":"senseval2.d001.s007.t000","gold_sense_keys":["today%1:28:00::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"today%1:28:00::","call_ids":["senseval2.d001.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"today%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.0039900000000000005,"input_uncached_usd":0.0019500000000000001,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.036266041919589}
{"item_id":"senseval2.d001.s007.t001","gold_sense_keys":["know%2:31:01::","know%2:31:02::"],"candidates":[{"index":1,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":2,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":3,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":4,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":5,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":6,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":7,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":8,"sense_key":"know%2:31:14::","synset_id":"know.v.11"},{"index":9,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":10,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":11,"sense_key":"know%2:31:15::","synset_id":"know.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d001.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":828,"cached_input_tokens":0,"output_tokens":3118,"reasoning_output_tokens":3072},"cost":{"currency":"USD","total_usd":0.09768000000000002,"input_uncached_usd":0.0041400000000000005,"input_cached_usd":0.0,"output_usd":0.09354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":57.383519374998286}
{"item_id":"senseval2.d001.s007.t002","gold_sense_keys":["accumulation%1:22:00::"],"candidates":[{"index":1,"sense_key":"accumulation%1:22:00::","synset_id":"accretion.n.01"},{"index":2,"sense_key":"accumulation%1:21:00::","synset_id":"accumulation.n.04"},{"index":3,"sense_key":"accumulation%1:04:00::","synset_id":"accumulation.n.03"},{"index":4,"sense_key":"accumulation%1:14:00::","synset_id":"collection.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"accumulation%1:22:00::","call_ids":["senseval2.d001.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"accumulation%1:22:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":718,"reasoning_output_tokens":696},"cost":{"currency":"USD","total_usd":0.023815,"input_uncached_usd":0.002275,"input_cached_usd":0.0,"output_usd":0.02154,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":15.649096000008285}
{"item_id":"senseval2.d001.s007.t003","gold_sense_keys":["altered%3:00:00::"],"candidates":[{"index":1,"sense_key":"altered%5:00:00:modified:00","synset_id":"adapted.s.01"},{"index":2,"sense_key":"altered%5:00:00:castrated:00","synset_id":"altered.s.01"},{"index":3,"sense_key":"altered%3:00:00::","synset_id":"altered.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"altered%3:00:00::","call_ids":["senseval2.d001.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"altered%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.004375,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7024416669737548}
{"item_id":"senseval2.d001.s007.t005","gold_sense_keys":["initiate%2:36:01::"],"candidates":[{"index":1,"sense_key":"initiate%2:41:01::","synset_id":"initiate.v.03"},{"index":2,"sense_key":"initiate%2:36:00::","synset_id":"initiate.v.02"},{"index":3,"sense_key":"initiate%2:36:01::","synset_id":"originate.v.02"},{"index":4,"sense_key":"initiate%2:32:00::","synset_id":"broach.v.01"},{"index":5,"sense_key":"initiate%2:30:00::","synset_id":"lead_up.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"initiate%2:30:00::","call_ids":["senseval2.d001.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"initiate%2:30:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":1035,"reasoning_output_tokens":1013},"cost":{"currency":"USD","total_usd":0.03367,"input_uncached_usd":0.0026200000000000004,"input_cached_usd":0.0,"output_usd":0.03105,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":18.82452049991116}
{"item_id":"senseval2.d001.s007.t006","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":2,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":3,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s007.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00446,"input_uncached_usd":0.0025700000000000002,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0396602500695735}
{"item_id":"senseval2.d001.s007.t007","gold_sense_keys":["then%4:02:00::","then%4:02:02::"],"candidates":[{"index":1,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":2,"sense_key":"then%4:02:01::","synset_id":"then.r.03"},{"index":3,"sense_key":"then%4:02:00::","synset_id":"then.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"then%4:02:00::","call_ids":["senseval2.d001.s007.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"then%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004215,"input_uncached_usd":0.0022050000000000004,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.075943458941765}
{"item_id":"senseval2.d001.s007.t008","gold_sense_keys":["propel%2:35:00::"],"candidates":[{"index":1,"sense_key":"propel%2:36:00::","synset_id":"motivate.v.01"},{"index":2,"sense_key":"propel%2:35:00::","synset_id":"propel.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"propel%2:35:00::","call_ids":["senseval2.d001.s007.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"propel%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":399,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":148},"cost":{"currency":"USD","total_usd":0.0068850000000000005,"input_uncached_usd":0.0019950000000000002,"input_cached_usd":0.0,"output_usd":0.00489,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.217208457994275}
{"item_id":"senseval2.d001.s007.t009","gold_sense_keys":["deadly%5:00:01:fatal:00"],"candidates":[{"index":1,"sense_key":"deadly%5:00:00:toxic:00","synset_id":"deadly.s.03"},{"index":2,"sense_key":"deadly%5:00:01:fatal:00","synset_id":"deadly.s.01"},{"index":3,"sense_key":"deadly%5:00:00:noxious:00","synset_id":"baneful.s.01"},{"index":4,"sense_key":"deadly%5:00:02:fatal:00","synset_id":"deadly.s.02"},{"index":5,"sense_key":"deadly%5:00:00:virulent:00","synset_id":"deadly.s.06"},{"index":6,"sense_key":"deadly%5:00:00:unpardonable:00","synset_id":"deadly.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deadly%5:00:01:fatal:00","call_ids":["senseval2.d001.s007.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deadly%5:00:01:fatal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":566,"cached_input_tokens":0,"output_tokens":231,"reasoning_output_tokens":215},"cost":{"currency":"USD","total_usd":0.009760000000000001,"input_uncached_usd":0.00283,"input_cached_usd":0.0,"output_usd":0.00693,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.864256875007413}
{"item_id":"senseval2.d001.s007.t010","gold_sense_keys":["state%1:03:00::"],"candidates":[{"index":1,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":2,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":6,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"},{"index":7,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":8,"sense_key":"state%1:15:00::","synset_id":"country.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"state%1:03:00::","call_ids":["senseval2.d001.s007.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"state%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":704,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.00598,"input_uncached_usd":0.00352,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.34947983303573}
{"item_id":"senseval2.d001.s008.t001","gold_sense_keys":["call%2:32:00::"],"candidates":[{"index":1,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":2,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":3,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":4,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":5,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":6,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":7,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":8,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":9,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":10,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":11,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":12,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":13,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":14,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":15,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":16,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":17,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":18,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":19,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":20,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":21,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":22,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":23,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":24,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":25,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":26,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":27,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":28,"sense_key":"call%2:29:05::","synset_id":"call.v.28"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":23,"chosen_sense_key":"call%2:32:00::","call_ids":["senseval2.d001.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":23,"predicted_sense_key":"call%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1485,"cached_input_tokens":0,"output_tokens":433,"reasoning_output_tokens":417},"cost":{"currency":"USD","total_usd":0.020415,"input_uncached_usd":0.007425,"input_cached_usd":0.0,"output_usd":0.01299,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.896646125009283}
{"item_id":"senseval2.d001.s008.t002","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":2,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":3,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":4,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":5,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"},{"index":6,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":7,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":8,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":9,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d001.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":791,"cached_input_tokens":0,"output_tokens":158,"reasoning_output_tokens":142},"cost":{"currency":"USD","total_usd":0.008695000000000001,"input_uncached_usd":0.003955,"input_cached_usd":0.0,"output_usd":0.00474,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4266280829906464}
{"item_id":"senseval2.d001.s008.t003","gold_sense_keys":["class%1:14:00::"],"candidates":[{"index":1,"sense_key":"class%1:07:00::","synset_id":"class.n.08"},{"index":2,"sense_key":"class%1:14:01::","synset_id":"class.n.03"},{"index":3,"sense_key":"class%1:14:04::","synset_id":"class.n.06"},{"index":4,"sense_key":"class%1:14:02::","synset_id":"class.n.07"},{"index":5,"sense_key":"class%1:04:00::","synset_id":"course.n.01"},{"index":6,"sense_key":"class%1:14:05::","synset_id":"class.n.05"},{"index":7,"sense_key":"class%1:14:03::","synset_id":"class.n.02"},{"index":8,"sense_key":"class%1:14:00::","synset_id":"class.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"class%1:14:00::","call_ids":["senseval2.d001.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"class%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":646,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.005180000000000001,"input_uncached_usd":0.0032300000000000002,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3469960000365973}
{"item_id":"senseval2.d001.s008.t005","gold_sense_keys":["simply%4:02:01::"],"candidates":[{"index":1,"sense_key":"simply%4:02:03::","synset_id":"just.r.04"},{"index":2,"sense_key":"simply%4:02:02::","synset_id":"simply.r.03"},{"index":3,"sense_key":"simply%4:02:01::","synset_id":"plainly.r.02"},{"index":4,"sense_key":"simply%4:02:00::","synset_id":"merely.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"simply%4:02:01::","call_ids":["senseval2.d001.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"simply%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":496,"reasoning_output_tokens":480},"cost":{"currency":"USD","total_usd":0.017235,"input_uncached_usd":0.0023550000000000003,"input_cached_usd":0.0,"output_usd":0.01488,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.570975082926452}
{"item_id":"senseval2.d001.s009.t000","gold_sense_keys":["function%2:35:00::"],"candidates":[{"index":1,"sense_key":"function%2:42:12::","synset_id":"serve.v.01"},{"index":2,"sense_key":"function%2:35:00::","synset_id":"function.v.01"},{"index":3,"sense_key":"function%2:33:00::","synset_id":"officiate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"function%2:35:00::","call_ids":["senseval2.d001.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"function%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":85},"cost":{"currency":"USD","total_usd":0.00542,"input_uncached_usd":0.0024200000000000003,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2450494579970837}
{"item_id":"senseval2.d001.s009.t004","gold_sense_keys":["hold%2:41:15::"],"candidates":[{"index":1,"sense_key":"hold%2:31:04::","synset_id":"hold.v.22"},{"index":2,"sense_key":"hold%2:35:02::","synset_id":"hold.v.14"},{"index":3,"sense_key":"hold%2:34:00::","synset_id":"carry.v.33"},{"index":4,"sense_key":"hold%2:33:05::","synset_id":"defend.v.03"},{"index":5,"sense_key":"hold%2:40:02::","synset_id":"retain.v.03"},{"index":6,"sense_key":"hold%2:35:09::","synset_id":"hold.v.31"},{"index":7,"sense_key":"hold%2:35:03::","synset_id":"restrain.v.03"},{"index":8,"sense_key":"hold%2:37:00::","synset_id":"harbor.v.01"},{"index":9,"sense_key":"hold%2:42:05::","synset_id":"accommodate.v.04"},{"index":10,"sense_key":"hold%2:42:02::","synset_id":"prevail.v.02"},{"index":11,"sense_key":"hold%2:31:01::","synset_id":"deem.v.01"},{"index":12,"sense_key":"hold%2:42:04::","synset_id":"apply.v.02"},{"index":13,"sense_key":"hold%2:32:13::","synset_id":"oblige.v.02"},{"index":14,"sense_key":"hold%2:32:11::","synset_id":"declare.v.04"},{"index":15,"sense_key":"hold%2:42:13::","synset_id":"hold.v.11"},{"index":16,"sense_key":"hold%2:42:01::","synset_id":"hold.v.13"},{"index":17,"sense_key":"hold%2:32:15::","synset_id":"agree.v.01"},{"index":18,"sense_key":"hold%2:42:12::","synset_id":"hold.v.26"},{"index":19,"sense_key":"hold%2:33:00::","synset_id":"hold.v.33"},{"index":20,"sense_key":"hold%2:31:10::","synset_id":"hold.v.23"},{"index":21,"sense_key":"hold%2:31:08::","synset_id":"hold.v.16"},{"index":22,"sense_key":"hold%2:35:01::","synset_id":"hold.v.10"},{"index":23,"sense_key":"hold%2:35:00::","synset_id":"hold.v.02"},{"index":24,"sense_key":"hold%2:36:00::","synset_id":"hold.v.03"},{"index":25,"sense_key":"hold%2:38:05::","synset_id":"halt.v.01"},{"index":26,"sense_key":"hold%2:42:00::","synset_id":"keep.v.01"},{"index":27,"sense_key":"hold%2:40:00::","synset_id":"have.v.01"},{"index":28,"sense_key":"hold%2:41:13::","synset_id":"hold.v.28"},{"index":29,"sense_key":"hold%2:41:15::","synset_id":"control.v.02"},{"index":30,"sense_key":"hold%2:29:04::","synset_id":"hold.v.36"},{"index":31,"sense_key":"hold%2:42:08::","synset_id":"defy.v.01"},{"index":32,"sense_key":"hold%2:41:14::","synset_id":"hold.v.29"},{"index":33,"sense_key":"hold%2:42:14::","synset_id":"contain.v.05"},{"index":34,"sense_key":"hold%2:40:04::","synset_id":"bear.v.11"},{"index":35,"sense_key":"hold%2:42:07::","synset_id":"hold.v.17"},{"index":36,"sense_key":"hold%2:41:00::","synset_id":"reserve.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":29,"chosen_sense_key":"hold%2:41:15::","call_ids":["senseval2.d001.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":29,"predicted_sense_key":"hold%2:41:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1968,"cached_input_tokens":0,"output_tokens":168,"reasoning_output_tokens":152},"cost":{"currency":"USD","total_usd":0.01488,"input_uncached_usd":0.009840000000000002,"input_cached_usd":0.0,"output_usd":0.00504,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.462228166987188}
{"item_id":"senseval2.d001.s009.t005","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":2,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":3,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":4,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":5,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":6,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":7,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.00451,"input_uncached_usd":0.0030700000000000002,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.429945332929492}
{"item_id":"senseval2.d001.s009.t006","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":2,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":3,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"},{"index":4,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":5,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":6,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":7,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"growth%1:22:02::","call_ids":["senseval2.d001.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"growth%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01901,"input_uncached_usd":0.00317,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.765308999922127}
{"item_id":"senseval2.d001.s010.t001","gold_sense_keys":["damage%2:30:00::"],"candidates":[{"index":1,"sense_key":"damage%2:30:00::","synset_id":"damage.v.01"},{"index":2,"sense_key":"damage%2:30:01::","synset_id":"damage.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%2:30:00::","call_ids":["senseval2.d001.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":137},"cost":{"currency":"USD","total_usd":0.006575,"input_uncached_usd":0.0020150000000000003,"input_cached_usd":0.0,"output_usd":0.00456,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3151017910568044}
{"item_id":"senseval2.d001.s010.t003","gold_sense_keys":["radiation%1:19:00::"],"candidates":[{"index":1,"sense_key":"radiation%1:22:00::","synset_id":"radiation.n.04"},{"index":2,"sense_key":"radiation%1:11:00::","synset_id":"radiation.n.05"},{"index":3,"sense_key":"radiation%1:26:00::","synset_id":"radiation_sickness.n.01"},{"index":4,"sense_key":"radiation%1:19:00::","synset_id":"radiation.n.01"},{"index":5,"sense_key":"radiation%1:04:00::","synset_id":"radiation.n.02"},{"index":6,"sense_key":"radiation%1:08:00::","synset_id":"radiation.n.06"},{"index":7,"sense_key":"radiation%1:04:01::","synset_id":"radiotherapy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"radiation%1:19:00::","call_ids":["senseval2.d001.s010.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"radiation%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.00576,"input_uncached_usd":0.0031500000000000005,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.8826575409621}
{"item_id":"senseval2.d001.s010.t011","gold_sense_keys":["work%2:35:02::","work%2:41:03::"],"candidates":[{"index":1,"sense_key":"work%2:41:12::","synset_id":"work.v.17"},{"index":2,"sense_key":"work%2:35:14::","synset_id":"work.v.21"},{"index":3,"sense_key":"work%2:30:13::","synset_id":"sour.v.01"},{"index":4,"sense_key":"work%2:41:02::","synset_id":"work.v.01"},{"index":5,"sense_key":"work%2:36:09::","synset_id":"cultivate.v.02"},{"index":6,"sense_key":"work%2:37:10::","synset_id":"work.v.19"},{"index":7,"sense_key":"work%2:41:11::","synset_id":"work.v.16"},{"index":8,"sense_key":"work%2:41:04::","synset_id":"work.v.08"},{"index":9,"sense_key":"work%2:38:02::","synset_id":"work.v.10"},{"index":10,"sense_key":"work%2:36:12::","synset_id":"shape.v.02"},{"index":11,"sense_key":"work%2:30:10::","synset_id":"work.v.27"},{"index":12,"sense_key":"work%2:41:05::","synset_id":"work.v.09"},{"index":13,"sense_key":"work%2:42:10::","synset_id":"work.v.14"},{"index":14,"sense_key":"work%2:36:01::","synset_id":"bring.v.03"},{"index":15,"sense_key":"work%2:41:00::","synset_id":"work.v.02"},{"index":16,"sense_key":"work%2:41:10::","synset_id":"influence.v.01"},{"index":17,"sense_key":"work%2:30:14::","synset_id":"ferment.v.03"},{"index":18,"sense_key":"work%2:41:01::","synset_id":"work.v.12"},{"index":19,"sense_key":"work%2:34:10::","synset_id":"exploit.v.01"},{"index":20,"sense_key":"work%2:35:00::","synset_id":"knead.v.01"},{"index":21,"sense_key":"work%2:41:03::","synset_id":"work.v.03"},{"index":22,"sense_key":"work%2:39:10::","synset_id":"work.v.18"},{"index":23,"sense_key":"work%2:31:13::","synset_id":"solve.v.01"},{"index":24,"sense_key":"work%2:36:00::","synset_id":"work.v.05"},{"index":25,"sense_key":"work%2:35:02::","synset_id":"function.v.01"},{"index":26,"sense_key":"work%2:38:00::","synset_id":"make.v.36"},{"index":27,"sense_key":"work%2:29:00::","synset_id":"exercise.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":25,"chosen_sense_key":"work%2:35:02::","call_ids":["senseval2.d001.s010.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":25,"predicted_sense_key":"work%2:35:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1505,"cached_input_tokens":0,"output_tokens":334,"reasoning_output_tokens":318},"cost":{"currency":"USD","total_usd":0.017544999999999998,"input_uncached_usd":0.0075250000000000004,"input_cached_usd":0.0,"output_usd":0.01002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.123888666974381}
{"item_id":"senseval2.d001.s010.t012","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":2,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":3,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":4,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":5,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":6,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s010.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":41,"reasoning_output_tokens":26},"cost":{"currency":"USD","total_usd":0.00425,"input_uncached_usd":0.00302,"input_cached_usd":0.0,"output_usd":0.00123,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8993344579357654}
{"item_id":"senseval2.d001.s010.t014","gold_sense_keys":["turn%2:30:03::","turn%2:42:00::"],"candidates":[{"index":1,"sense_key":"turn%2:35:11::","synset_id":"flex.v.05"},{"index":2,"sense_key":"turn%2:30:03::","synset_id":"turn.v.07"},{"index":3,"sense_key":"turn%2:30:14::","synset_id":"turn.v.26"},{"index":4,"sense_key":"turn%2:38:06::","synset_id":"turn.v.11"},{"index":5,"sense_key":"turn%2:35:10::","synset_id":"turn.v.22"},{"index":6,"sense_key":"turn%2:41:10::","synset_id":"turn.v.17"},{"index":7,"sense_key":"turn%2:38:13::","synset_id":"turn.v.19"},{"index":8,"sense_key":"turn%2:30:01::","synset_id":"sour.v.01"},{"index":9,"sense_key":"turn%2:32:14::","synset_id":"call_on.v.01"},{"index":10,"sense_key":"turn%2:30:09::","synset_id":"turn.v.14"},{"index":11,"sense_key":"turn%2:38:04::","synset_id":"turn.v.09"},{"index":12,"sense_key":"turn%2:38:14::","synset_id":"turn.v.20"},{"index":13,"sense_key":"turn%2:38:00::","synset_id":"turn.v.01"},{"index":14,"sense_key":"turn%2:35:08::","synset_id":"turn.v.08"},{"index":15,"sense_key":"turn%2:38:01::","synset_id":"turn.v.10"},{"index":16,"sense_key":"turn%2:30:04::","synset_id":"change_state.v.01"},{"index":17,"sense_key":"turn%2:29:00::","synset_id":"twist.v.10"},{"index":18,"sense_key":"turn%2:35:06::","synset_id":"turn.v.13"},{"index":19,"sense_key":"turn%2:38:03::","synset_id":"turn.v.06"},{"index":20,"sense_key":"turn%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":21,"sense_key":"turn%2:33:13::","synset_id":"turn.v.23"},{"index":22,"sense_key":"turn%2:38:02::","synset_id":"turn.v.04"},{"index":23,"sense_key":"turn%2:40:13::","synset_id":"turn.v.18"},{"index":24,"sense_key":"turn%2:42:13::","synset_id":"turn.v.16"},{"index":25,"sense_key":"turn%2:36:00::","synset_id":"plow.v.01"},{"index":26,"sense_key":"turn%2:42:00::","synset_id":"become.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"turn%2:30:03::","call_ids":["senseval2.d001.s010.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"turn%2:30:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1455,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":115},"cost":{"currency":"USD","total_usd":0.011205,"input_uncached_usd":0.007275,"input_cached_usd":0.0,"output_usd":0.00393,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0141236670315266}
{"item_id":"senseval2.d001.s011.t001","gold_sense_keys":["identify%2:31:01::"],"candidates":[{"index":1,"sense_key":"identify%2:32:00::","synset_id":"name.v.02"},{"index":2,"sense_key":"identify%2:31:01::","synset_id":"identify.v.05"},{"index":3,"sense_key":"identify%2:31:04::","synset_id":"identify.v.03"},{"index":4,"sense_key":"identify%2:31:02::","synset_id":"identify.v.06"},{"index":5,"sense_key":"identify%2:31:00::","synset_id":"identify.v.01"},{"index":6,"sense_key":"identify%2:31:03::","synset_id":"identify.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"identify%2:31:01::","call_ids":["senseval2.d001.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"identify%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":135},"cost":{"currency":"USD","total_usd":0.00746,"input_uncached_usd":0.0029600000000000004,"input_cached_usd":0.0,"output_usd":0.0045000000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.598044457961805}
{"item_id":"senseval2.d001.s011.t003","gold_sense_keys":["differ%2:42:00::"],"candidates":[{"index":1,"sense_key":"differ%2:32:00::","synset_id":"disagree.v.01"},{"index":2,"sense_key":"differ%2:42:00::","synset_id":"differ.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"differ%2:42:00::","call_ids":["senseval2.d001.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"differ%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.00369,"input_uncached_usd":0.0020700000000000002,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0823513750219718}
{"item_id":"senseval2.d001.s011.t004","gold_sense_keys":["family%1:14:04::"],"candidates":[{"index":1,"sense_key":"family%1:14:04::","synset_id":"class.n.01"},{"index":2,"sense_key":"family%1:14:06::","synset_id":"syndicate.n.01"},{"index":3,"sense_key":"family%1:14:05::","synset_id":"family.n.08"},{"index":4,"sense_key":"family%1:18:00::","synset_id":"kin.n.01"},{"index":5,"sense_key":"family%1:14:01::","synset_id":"family.n.04"},{"index":6,"sense_key":"family%1:14:00::","synset_id":"family.n.02"},{"index":7,"sense_key":"family%1:14:02::","synset_id":"family.n.01"},{"index":8,"sense_key":"family%1:14:03::","synset_id":"family.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"family%1:14:04::","call_ids":["senseval2.d001.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"family%1:14:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":703,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":88},"cost":{"currency":"USD","total_usd":0.006605,"input_uncached_usd":0.0035150000000000003,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3434514169348404}
{"item_id":"senseval2.d001.s011.t006","gold_sense_keys":["discover%2:31:00::","discover%2:31:03::","discover%2:36:00::","discover%2:39:03::"],"candidates":[{"index":1,"sense_key":"discover%2:31:01::","synset_id":"learn.v.02"},{"index":2,"sense_key":"discover%2:39:03::","synset_id":"detect.v.01"},{"index":3,"sense_key":"discover%2:31:00::","synset_id":"discover.v.04"},{"index":4,"sense_key":"discover%2:31:03::","synset_id":"identify.v.05"},{"index":5,"sense_key":"discover%2:39:00::","synset_id":"discover.v.07"},{"index":6,"sense_key":"discover%2:36:00::","synset_id":"discover.v.03"},{"index":7,"sense_key":"discover%2:32:04::","synset_id":"unwrap.v.02"},{"index":8,"sense_key":"discover%2:40:00::","synset_id":"fall_upon.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"discover%2:36:00::","call_ids":["senseval2.d001.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"discover%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":769,"cached_input_tokens":0,"output_tokens":299,"reasoning_output_tokens":283},"cost":{"currency":"USD","total_usd":0.012815,"input_uncached_usd":0.0038450000000000003,"input_cached_usd":0.0,"output_usd":0.00897,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.657032749964856}
{"item_id":"senseval2.d001.s011.t007","gold_sense_keys":["early%3:00:00::"],"candidates":[{"index":1,"sense_key":"early%3:00:01::","synset_id":"early.a.05"},{"index":2,"sense_key":"early%5:00:01:future:00","synset_id":"early.s.03"},{"index":3,"sense_key":"early%3:00:02::","synset_id":"early.a.02"},{"index":4,"sense_key":"early%3:00:00::","synset_id":"early.a.01"},{"index":5,"sense_key":"early%5:00:00:past:00","synset_id":"early.s.01"},{"index":6,"sense_key":"early%5:00:00:young:00","synset_id":"early.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"early%3:00:00::","call_ids":["senseval2.d001.s011.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"early%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.005215000000000001,"input_uncached_usd":0.0029950000000000003,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1688013749662787}
{"item_id":"senseval2.d001.s011.t008","gold_sense_keys":["call%2:32:02::"],"candidates":[{"index":1,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":2,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":3,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":4,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":5,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":6,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":7,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":8,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":9,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":10,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":11,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":12,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":13,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":14,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":15,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":16,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":17,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":18,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":19,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":20,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":21,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":22,"sense_key":"call%2:29:05::","synset_id":"call.v.28"},{"index":23,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":24,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":25,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":26,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":27,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":28,"sense_key":"call%2:31:05::","synset_id":"call.v.27"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"call%2:32:02::","call_ids":["senseval2.d001.s011.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"call%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1513,"cached_input_tokens":0,"output_tokens":276,"reasoning_output_tokens":260},"cost":{"currency":"USD","total_usd":0.015845,"input_uncached_usd":0.0075650000000000005,"input_cached_usd":0.0,"output_usd":0.008280000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.947494333027862}
{"item_id":"senseval2.d001.s012.t001","gold_sense_keys":["present%3:00:02::"],"candidates":[{"index":1,"sense_key":"present%3:00:02::","synset_id":"present.a.02"},{"index":2,"sense_key":"present%3:00:01::","synset_id":"present.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"present%3:00:02::","call_ids":["senseval2.d001.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"present%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.005135000000000001,"input_uncached_usd":0.0021950000000000003,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.239379084086977}
{"item_id":"senseval2.d001.s012.t002","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":4,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":5,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":6,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":7,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.00455,"input_uncached_usd":0.0031400000000000004,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5360779169714078}
{"item_id":"senseval2.d001.s012.t003","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":2,"sense_key":"become%2:42:03::","synset_id":"become.v.04"},{"index":3,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":4,"sense_key":"become%2:42:01::","synset_id":"become.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"become%2:30:00::","call_ids":["senseval2.d001.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"become%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":315,"reasoning_output_tokens":299},"cost":{"currency":"USD","total_usd":0.01205,"input_uncached_usd":0.0026000000000000003,"input_cached_usd":0.0,"output_usd":0.00945,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.532996707945131}
{"item_id":"senseval2.d001.s012.t006","gold_sense_keys":["find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":4,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":5,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":6,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":7,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":8,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":9,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":10,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":11,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":12,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"},{"index":13,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":14,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":15,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":16,"sense_key":"find%2:40:03::","synset_id":"find.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"find%2:39:02::","call_ids":["senseval2.d001.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"find%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1147,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":141},"cost":{"currency":"USD","total_usd":0.010445000000000001,"input_uncached_usd":0.0057350000000000005,"input_cached_usd":0.0,"output_usd":0.00471,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6129832919687033}
{"item_id":"senseval2.d001.s012.t007","gold_sense_keys":["normal%3:00:01::"],"candidates":[{"index":1,"sense_key":"normal%3:00:01::","synset_id":"normal.a.01"},{"index":2,"sense_key":"normal%5:00:00:perpendicular:00","synset_id":"normal.s.01"},{"index":3,"sense_key":"normal%3:00:02::","synset_id":"normal.a.02"},{"index":4,"sense_key":"normal%3:00:03::","synset_id":"normal.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"normal%3:00:01::","call_ids":["senseval2.d001.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"normal%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.005640000000000001,"input_uncached_usd":0.0024600000000000004,"input_cached_usd":0.0,"output_usd":0.00318,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5681988750584424}
{"item_id":"senseval2.d001.s012.t008","gold_sense_keys":["cancerous%5:00:00:malignant:02"],"candidates":[{"index":1,"sense_key":"cancerous%5:00:00:malign:00","synset_id":"cancerous.s.02"},{"index":2,"sense_key":"cancerous%5:00:00:malignant:02","synset_id":"cancerous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cancerous%5:00:00:malignant:02","call_ids":["senseval2.d001.s012.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cancerous%5:00:00:malignant:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.003735,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.447590333991684}
{"item_id":"senseval2.d001.s012.t009","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":2,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":3,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":4,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":5,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":6,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":7,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s012.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":627,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.005025,"input_uncached_usd":0.003135,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2190631669946015}
{"item_id":"senseval2.d001.s012.t010","gold_sense_keys":["suggest%2:32:02::"],"candidates":[{"index":1,"sense_key":"suggest%2:32:01::","synset_id":"suggest.v.05"},{"index":2,"sense_key":"suggest%2:32:02::","synset_id":"suggest.v.03"},{"index":3,"sense_key":"suggest%2:32:04::","synset_id":"hint.v.01"},{"index":4,"sense_key":"suggest%2:32:00::","synset_id":"propose.v.01"},{"index":5,"sense_key":"suggest%2:32:03::","synset_id":"indicate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"suggest%2:32:02::","call_ids":["senseval2.d001.s012.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"suggest%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.0052450000000000005,"input_uncached_usd":0.0027250000000000004,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9354679589159787}
{"item_id":"senseval2.d001.s012.t012","gold_sense_keys":["cause%2:36:00::"],"candidates":[{"index":1,"sense_key":"cause%2:32:00::","synset_id":"induce.v.02"},{"index":2,"sense_key":"cause%2:36:00::","synset_id":"cause.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cause%2:36:00::","call_ids":["senseval2.d001.s012.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cause%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.004185,"input_uncached_usd":0.0023250000000000002,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.186641833977774}
{"item_id":"senseval2.d001.s012.t013","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":3,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":4,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":5,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s012.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.00434,"input_uncached_usd":0.0027800000000000004,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.943257583072409}
{"item_id":"senseval2.d001.s013.t000","gold_sense_keys":["recent%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"recent%5:00:00:new:00","synset_id":"recent.s.01"},{"index":2,"sense_key":"recent%5:00:00:past:00","synset_id":"late.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"recent%5:00:00:past:00","call_ids":["senseval2.d001.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"recent%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00388,"input_uncached_usd":0.0021100000000000003,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8419676250778139}
{"item_id":"senseval2.d001.s013.t001","gold_sense_keys":["month%1:28:01::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["senseval2.d001.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":421,"cached_input_tokens":0,"output_tokens":1484,"reasoning_output_tokens":1456},"cost":{"currency":"USD","total_usd":0.04662500000000001,"input_uncached_usd":0.002105,"input_cached_usd":0.0,"output_usd":0.044520000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":25.969411792000756}
{"item_id":"senseval2.d001.s013.t003","gold_sense_keys":["come%2:30:03::"],"candidates":[{"index":1,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":2,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":3,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":4,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":5,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":6,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":7,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":8,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":9,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":10,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":11,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":12,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":13,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":14,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":15,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":16,"sense_key":"come%2:31:13::","synset_id":"come.v.21"},{"index":17,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":18,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":19,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":20,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":21,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"come%2:30:03::","call_ids":["senseval2.d001.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"come%2:30:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1219,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.008405000000000001,"input_uncached_usd":0.006095000000000001,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9627983749378473}
{"item_id":"senseval2.d001.s013.t006","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":4,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":5,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s013.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.004985,"input_uncached_usd":0.002735,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.115768249961548}
{"item_id":"senseval2.d001.s013.t011","gold_sense_keys":["turn%2:42:13::"],"candidates":[{"index":1,"sense_key":"turn%2:30:09::","synset_id":"turn.v.14"},{"index":2,"sense_key":"turn%2:38:00::","synset_id":"turn.v.01"},{"index":3,"sense_key":"turn%2:42:00::","synset_id":"become.v.02"},{"index":4,"sense_key":"turn%2:30:14::","synset_id":"turn.v.26"},{"index":5,"sense_key":"turn%2:38:01::","synset_id":"turn.v.10"},{"index":6,"sense_key":"turn%2:33:13::","synset_id":"turn.v.23"},{"index":7,"sense_key":"turn%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":8,"sense_key":"turn%2:41:10::","synset_id":"turn.v.17"},{"index":9,"sense_key":"turn%2:30:03::","synset_id":"turn.v.07"},{"index":10,"sense_key":"turn%2:35:11::","synset_id":"flex.v.05"},{"index":11,"sense_key":"turn%2:35:06::","synset_id":"turn.v.13"},{"index":12,"sense_key":"turn%2:38:04::","synset_id":"turn.v.09"},{"index":13,"sense_key":"turn%2:30:01::","synset_id":"sour.v.01"},{"index":14,"sense_key":"turn%2:38:03::","synset_id":"turn.v.06"},{"index":15,"sense_key":"turn%2:29:00::","synset_id":"twist.v.10"},{"index":16,"sense_key":"turn%2:38:02::","synset_id":"turn.v.04"},{"index":17,"sense_key":"turn%2:35:10::","synset_id":"turn.v.22"},{"index":18,"sense_key":"turn%2:38:06::","synset_id":"turn.v.11"},{"index":19,"sense_key":"turn%2:30:04::","synset_id":"change_state.v.01"},{"index":20,"sense_key":"turn%2:40:13::","synset_id":"turn.v.18"},{"index":21,"sense_key":"turn%2:36:00::","synset_id":"plow.v.01"},{"index":22,"sense_key":"turn%2:35:08::","synset_id":"turn.v.08"},{"index":23,"sense_key":"turn%2:38:13::","synset_id":"turn.v.19"},{"index":24,"sense_key":"turn%2:42:13::","synset_id":"turn.v.16"},{"index":25,"sense_key":"turn%2:32:14::","synset_id":"call_on.v.01"},{"index":26,"sense_key":"turn%2:38:14::","synset_id":"turn.v.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":24,"chosen_sense_key":"turn%2:42:13::","call_ids":["senseval2.d001.s013.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":24,"predicted_sense_key":"turn%2:42:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1470,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":94},"cost":{"currency":"USD","total_usd":0.01065,"input_uncached_usd":0.007350000000000001,"input_cached_usd":0.0,"output_usd":0.0033,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.695876209065318}
{"item_id":"senseval2.d001.s013.t013","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":2,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":5,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":6,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":7,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s013.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.005045,"input_uncached_usd":0.003095,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3188824580283836}
{"item_id":"senseval2.d001.s013.t015","gold_sense_keys":["only%4:02:03::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":3,"sense_key":"only%4:02:05::","synset_id":"only.r.07"},{"index":4,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":5,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":6,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":7,"sense_key":"only%4:02:04::","synset_id":"only.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"only%4:02:03::","call_ids":["senseval2.d001.s013.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"only%4:02:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":109},"cost":{"currency":"USD","total_usd":0.006880000000000001,"input_uncached_usd":0.0031300000000000004,"input_cached_usd":0.0,"output_usd":0.0037500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2270446670008823}
{"item_id":"senseval2.d001.s013.t017","gold_sense_keys":["damage%2:30:00::"],"candidates":[{"index":1,"sense_key":"damage%2:30:01::","synset_id":"damage.v.02"},{"index":2,"sense_key":"damage%2:30:00::","synset_id":"damage.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"damage%2:30:00::","call_ids":["senseval2.d001.s013.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"damage%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":256,"reasoning_output_tokens":240},"cost":{"currency":"USD","total_usd":0.009765,"input_uncached_usd":0.002085,"input_cached_usd":0.0,"output_usd":0.00768,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.981867166003212}
{"item_id":"senseval2.d001.s014.t002","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"},{"index":2,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"},{"index":3,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.003705,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6559335420606658}
{"item_id":"senseval2.d001.s014.t003","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":3,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"},{"index":4,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.00544,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.409516124986112}
{"item_id":"senseval2.d001.s014.t004","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d001.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017945000000000003,"input_uncached_usd":0.002135,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.814004292013124}
{"item_id":"senseval2.d001.s015.t000","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":2,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"},{"index":3,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":4,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":94},"cost":{"currency":"USD","total_usd":0.00592,"input_uncached_usd":0.00265,"input_cached_usd":0.0,"output_usd":0.00327,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.886206416063942}
{"item_id":"senseval2.d001.s015.t003","gold_sense_keys":["need%2:42:00::"],"candidates":[{"index":1,"sense_key":"need%2:34:00::","synset_id":"want.v.02"},{"index":2,"sense_key":"need%2:42:00::","synset_id":"necessitate.v.01"},{"index":3,"sense_key":"need%2:34:01::","synset_id":"need.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"need%2:42:00::","call_ids":["senseval2.d001.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"need%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":1321,"reasoning_output_tokens":1293},"cost":{"currency":"USD","total_usd":0.042159999999999996,"input_uncached_usd":0.00253,"input_cached_usd":0.0,"output_usd":0.03963,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":23.30622820800636}
{"item_id":"senseval2.d001.s015.t004","gold_sense_keys":["control%2:41:01::"],"candidates":[{"index":1,"sense_key":"control%2:31:11::","synset_id":"control.v.06"},{"index":2,"sense_key":"control%2:31:00::","synset_id":"master.v.04"},{"index":3,"sense_key":"control%2:41:00::","synset_id":"control.v.01"},{"index":4,"sense_key":"control%2:31:01::","synset_id":"see.v.10"},{"index":5,"sense_key":"control%2:37:06::","synset_id":"manipulate.v.05"},{"index":6,"sense_key":"control%2:35:00::","synset_id":"operate.v.03"},{"index":7,"sense_key":"control%2:41:01::","synset_id":"control.v.02"},{"index":8,"sense_key":"control%2:41:02::","synset_id":"control.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"control%2:41:01::","call_ids":["senseval2.d001.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"control%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":168,"reasoning_output_tokens":152},"cost":{"currency":"USD","total_usd":0.008875000000000001,"input_uncached_usd":0.0038350000000000003,"input_cached_usd":0.0,"output_usd":0.00504,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.900994083029218}
{"item_id":"senseval2.d001.s015.t005","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":3,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":4,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":5,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.004765,"input_uncached_usd":0.003265,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6847854999359697}
{"item_id":"senseval2.d001.s015.t006","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":2,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":3,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":4,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":5,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":6,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"},{"index":7,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"growth%1:22:02::","call_ids":["senseval2.d001.s015.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"growth%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":203,"reasoning_output_tokens":187},"cost":{"currency":"USD","total_usd":0.00945,"input_uncached_usd":0.00336,"input_cached_usd":0.0,"output_usd":0.00609,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.997719125007279}
{"item_id":"senseval2.d001.s015.t007","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":581,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.004255,"input_uncached_usd":0.002905,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6420010419096798}
{"item_id":"senseval2.d001.s015.t008","gold_sense_keys":["arise%2:42:00::"],"candidates":[{"index":1,"sense_key":"arise%2:42:01::","synset_id":"arise.v.02"},{"index":2,"sense_key":"arise%2:29:00::","synset_id":"get_up.v.02"},{"index":3,"sense_key":"arise%2:38:00::","synset_id":"arise.v.03"},{"index":4,"sense_key":"arise%2:41:00::","synset_id":"rebel.v.01"},{"index":5,"sense_key":"arise%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"arise%2:38:04::","synset_id":"rise.v.01"},{"index":7,"sense_key":"arise%2:30:00::","synset_id":"arise.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"arise%2:42:00::","call_ids":["senseval2.d001.s015.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"arise%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.01551,"input_uncached_usd":0.00351,"input_cached_usd":0.0,"output_usd":0.012,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.386715415981598}
{"item_id":"senseval2.d001.s015.t009","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":2,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"},{"index":3,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":4,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s015.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.00523,"input_uncached_usd":0.00265,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.108489124919288}
{"item_id":"senseval2.d001.s015.t010","gold_sense_keys":["impaired%3:00:00::"],"candidates":[{"index":1,"sense_key":"impaired%3:00:00::","synset_id":"impaired.a.01"},{"index":2,"sense_key":"impaired%5:00:00:unfit:01","synset_id":"afflicted.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"impaired%3:00:00::","call_ids":["senseval2.d001.s015.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"impaired%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.00388,"input_uncached_usd":0.00223,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9960953330155462}
{"item_id":"senseval2.d001.s016.t000","gold_sense_keys":["person%1:03:00::"],"candidates":[{"index":1,"sense_key":"person%1:08:00::","synset_id":"person.n.02"},{"index":2,"sense_key":"person%1:03:00::","synset_id":"person.n.01"},{"index":3,"sense_key":"person%1:10:00::","synset_id":"person.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"person%1:03:00::","call_ids":["senseval2.d001.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"person%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.004240000000000001,"input_uncached_usd":0.00256,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4252375829964876}
{"item_id":"senseval2.d001.s016.t001","gold_sense_keys":["bear%2:29:01::"],"candidates":[{"index":1,"sense_key":"bear%2:29:01::","synset_id":"give_birth.v.01"},{"index":2,"sense_key":"bear%2:35:01::","synset_id":"hold.v.14"},{"index":3,"sense_key":"bear%2:35:00::","synset_id":"bear.v.04"},{"index":4,"sense_key":"bear%2:40:00::","synset_id":"bear.v.11"},{"index":5,"sense_key":"bear%2:29:02::","synset_id":"wear.v.02"},{"index":6,"sense_key":"bear%2:40:01::","synset_id":"yield.v.10"},{"index":7,"sense_key":"bear%2:40:02::","synset_id":"bear.v.06"},{"index":8,"sense_key":"bear%2:36:00::","synset_id":"bear.v.05"},{"index":9,"sense_key":"bear%2:29:00::","synset_id":"have_a_bun_in_the_oven.v.01"},{"index":10,"sense_key":"bear%2:41:00::","synset_id":"behave.v.02"},{"index":11,"sense_key":"bear%2:42:14::","synset_id":"hold.v.11"},{"index":12,"sense_key":"bear%2:42:01::","synset_id":"bear.v.01"},{"index":13,"sense_key":"bear%2:31:00::","synset_id":"digest.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bear%2:29:01::","call_ids":["senseval2.d001.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bear%2:29:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":949,"cached_input_tokens":0,"output_tokens":462,"reasoning_output_tokens":446},"cost":{"currency":"USD","total_usd":0.018605,"input_uncached_usd":0.004745,"input_cached_usd":0.0,"output_usd":0.01386,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.659731584019028}
{"item_id":"senseval2.d001.s016.t003","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"},{"index":2,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":3,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":4,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.0053100000000000005,"input_uncached_usd":0.0026100000000000003,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.452901958953589}
{"item_id":"senseval2.d001.s016.t005","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":2,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"},{"index":3,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":4,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.005130000000000001,"input_uncached_usd":0.0026100000000000003,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7281617920380086}
{"item_id":"senseval2.d001.s016.t006","gold_sense_keys":["damage%2:30:00::"],"candidates":[{"index":1,"sense_key":"damage%2:30:00::","synset_id":"damage.v.01"},{"index":2,"sense_key":"damage%2:30:01::","synset_id":"damage.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%2:30:00::","call_ids":["senseval2.d001.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":206,"reasoning_output_tokens":191},"cost":{"currency":"USD","total_usd":0.0084,"input_uncached_usd":0.00222,"input_cached_usd":0.0,"output_usd":0.00618,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.01209079194814}
{"item_id":"senseval2.d001.s016.t007","gold_sense_keys":["early%4:02:01::"],"candidates":[{"index":1,"sense_key":"early%4:02:02::","synset_id":"early.r.03"},{"index":2,"sense_key":"early%4:02:01::","synset_id":"early_on.r.01"},{"index":3,"sense_key":"early%4:02:00::","synset_id":"early.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"early%4:02:01::","call_ids":["senseval2.d001.s016.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"early%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.00493,"input_uncached_usd":0.0024400000000000003,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8739630000200123}
{"item_id":"senseval2.d001.s016.t008","gold_sense_keys":["life%1:28:00::","life%1:28:01::"],"candidates":[{"index":1,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":2,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":3,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":4,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":5,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":6,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":7,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":8,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":9,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":10,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":11,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"},{"index":12,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":13,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":14,"sense_key":"life%1:26:02::","synset_id":"life.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"life%1:28:00::","call_ids":["senseval2.d001.s016.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"life%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":939,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.036074999999999996,"input_uncached_usd":0.004695,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":19.269028916954994}
{"item_id":"senseval2.d001.s016.t009","gold_sense_keys":["especially%4:02:00::"],"candidates":[{"index":1,"sense_key":"especially%4:02:00::","synset_id":"particularly.r.01"},{"index":2,"sense_key":"especially%4:02:01::","synset_id":"specially.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"especially%4:02:00::","call_ids":["senseval2.d001.s016.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"especially%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.004105,"input_uncached_usd":0.002275,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6677910829894245}
{"item_id":"senseval2.d001.s016.t010","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":2,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":3,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":4,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":5,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s016.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.0043950000000000005,"input_uncached_usd":0.0028650000000000004,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.715295249945484}
{"item_id":"senseval2.d001.s016.t012","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:05::","synset_id":"only.r.07"},{"index":2,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":3,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":4,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":5,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":6,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":7,"sense_key":"only%4:02:04::","synset_id":"only.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"only%4:02:02::","call_ids":["senseval2.d001.s016.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"only%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":170,"reasoning_output_tokens":155},"cost":{"currency":"USD","total_usd":0.008360000000000001,"input_uncached_usd":0.0032600000000000003,"input_cached_usd":0.0,"output_usd":0.0051,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.147210208000615}
{"item_id":"senseval2.d001.s016.t013","gold_sense_keys":["lose%2:40:02::"],"candidates":[{"index":1,"sense_key":"lose%2:39:00::","synset_id":"lose.v.06"},{"index":2,"sense_key":"lose%2:40:02::","synset_id":"lose.v.01"},{"index":3,"sense_key":"lose%2:37:00::","synset_id":"lose.v.03"},{"index":4,"sense_key":"lose%2:40:00::","synset_id":"lose.v.08"},{"index":5,"sense_key":"lose%2:33:01::","synset_id":"fall_back.v.04"},{"index":6,"sense_key":"lose%2:33:00::","synset_id":"lose.v.02"},{"index":7,"sense_key":"lose%2:30:03::","synset_id":"suffer.v.11"},{"index":8,"sense_key":"lose%2:40:01::","synset_id":"lose.v.05"},{"index":9,"sense_key":"lose%2:40:06::","synset_id":"lose.v.07"},{"index":10,"sense_key":"lose%2:35:00::","synset_id":"misplace.v.01"},{"index":11,"sense_key":"lose%2:39:01::","synset_id":"miss.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"lose%2:40:02::","call_ids":["senseval2.d001.s016.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"lose%2:40:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":837,"cached_input_tokens":0,"output_tokens":167,"reasoning_output_tokens":151},"cost":{"currency":"USD","total_usd":0.009195000000000002,"input_uncached_usd":0.004185,"input_cached_usd":0.0,"output_usd":0.00501,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8569157919846475}
{"item_id":"senseval2.d001.s016.t014","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":2,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":3,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":4,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s016.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.00486,"input_uncached_usd":0.0027300000000000002,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7918235419783741}
{"item_id":"senseval2.d001.s016.t015","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s016.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":79},"cost":{"currency":"USD","total_usd":0.0054600000000000004,"input_uncached_usd":0.0026100000000000003,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.191626916988753}
{"item_id":"senseval2.d001.s016.t016","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":3,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s016.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.004605000000000001,"input_uncached_usd":0.0028650000000000004,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5908439999911934}
{"item_id":"senseval2.d001.s016.t017","gold_sense_keys":["develop%2:30:00::","develop%2:42:00::"],"candidates":[{"index":1,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":2,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":3,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":4,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":5,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":6,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":7,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":8,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":9,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":10,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":11,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"},{"index":12,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":13,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":14,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":15,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":16,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":17,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":18,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":19,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":20,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":21,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":16,"chosen_sense_key":"develop%2:42:00::","call_ids":["senseval2.d001.s016.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":16,"predicted_sense_key":"develop%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1312,"cached_input_tokens":0,"output_tokens":370,"reasoning_output_tokens":354},"cost":{"currency":"USD","total_usd":0.017660000000000002,"input_uncached_usd":0.006560000000000001,"input_cached_usd":0.0,"output_usd":0.0111,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.96180704201106}
{"item_id":"senseval2.d001.s017.t000","gold_sense_keys":["emerging%5:00:00:nascent:00"],"candidates":[{"index":1,"sense_key":"emerging%5:00:00:nascent:00","synset_id":"emergent.s.02"},{"index":2,"sense_key":"emerging%5:00:00:future:00","synset_id":"emerging.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"emerging%5:00:00:nascent:00","call_ids":["senseval2.d001.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"emerging%5:00:00:nascent:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.004765,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.078516999958083}
{"item_id":"senseval2.d001.s017.t001","gold_sense_keys":["genetic%3:01:00::"],"candidates":[{"index":1,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":2,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":3,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":4,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genetic%3:01:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":305,"reasoning_output_tokens":289},"cost":{"currency":"USD","total_usd":0.011865,"input_uncached_usd":0.0027150000000000004,"input_cached_usd":0.0,"output_usd":0.00915,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.260009457939304}
{"item_id":"senseval2.d001.s017.t003","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"},{"index":2,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"},{"index":3,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"},{"index":4,"sense_key":"able%3:00:00::","synset_id":"able.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"able%3:00:00::","call_ids":["senseval2.d001.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"able%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":418,"reasoning_output_tokens":402},"cost":{"currency":"USD","total_usd":0.015295,"input_uncached_usd":0.002755,"input_cached_usd":0.0,"output_usd":0.01254,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.428858291939832}
{"item_id":"senseval2.d001.s017.t006","gold_sense_keys":["individual%1:03:00::"],"candidates":[{"index":1,"sense_key":"individual%1:18:00::","synset_id":"individual.n.02"},{"index":2,"sense_key":"individual%1:03:00::","synset_id":"person.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"individual%1:03:00::","call_ids":["senseval2.d001.s017.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"individual%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.00447,"input_uncached_usd":0.0021300000000000004,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1793642080156133}
{"item_id":"senseval2.d001.s017.t009","gold_sense_keys":["believe%2:31:04::"],"candidates":[{"index":1,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"},{"index":2,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"},{"index":3,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"},{"index":4,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":5,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"believe%2:31:04::","call_ids":["senseval2.d001.s017.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"believe%2:31:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":288,"reasoning_output_tokens":272},"cost":{"currency":"USD","total_usd":0.01156,"input_uncached_usd":0.0029200000000000003,"input_cached_usd":0.0,"output_usd":0.00864,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.426796249928884}
{"item_id":"senseval2.d001.s017.t010","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":2,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":3,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":4,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":5,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":6,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":7,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":8,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"},{"index":9,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":10,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":11,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d001.s017.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":848,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02008,"input_uncached_usd":0.004240000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.293799875071272}
{"item_id":"senseval2.d001.s017.t011","gold_sense_keys":["age%1:28:02::"],"candidates":[{"index":1,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"},{"index":2,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":3,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":4,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"},{"index":5,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"age%1:28:02::","call_ids":["senseval2.d001.s017.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"age%1:28:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.004665,"input_uncached_usd":0.002925,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8731420839903876}
{"item_id":"senseval2.d001.s017.t013","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":2,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s017.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.004475,"input_uncached_usd":0.0028250000000000003,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9496086669387296}
{"item_id":"senseval2.d001.s018.t000","gold_sense_keys":["first%3:00:00::"],"candidates":[{"index":1,"sense_key":"first%5:00:00:best:00","synset_id":"first.s.04"},{"index":2,"sense_key":"first%5:00:02:opening:00","synset_id":"inaugural.s.01"},{"index":3,"sense_key":"first%3:00:00::","synset_id":"first.a.01"},{"index":4,"sense_key":"first%5:00:01:opening:00","synset_id":"beginning.s.01"},{"index":5,"sense_key":"first%3:00:03::","synset_id":"first.a.06"},{"index":6,"sense_key":"first%5:00:00:ordinal:00","synset_id":"first.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"first%3:00:00::","call_ids":["senseval2.d001.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"first%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":635,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.034554999999999995,"input_uncached_usd":0.0031750000000000003,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":19.997160624945536}
{"item_id":"senseval2.d001.s018.t001","gold_sense_keys":["beneficiary%1:18:00::"],"candidates":[{"index":1,"sense_key":"beneficiary%1:18:00::","synset_id":"beneficiary.n.01"},{"index":2,"sense_key":"beneficiary%1:10:00::","synset_id":"benefactive_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"beneficiary%1:18:00::","call_ids":["senseval2.d001.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"beneficiary%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.00421,"input_uncached_usd":0.0022600000000000003,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.847200792049989}
{"item_id":"senseval2.d001.s018.t002","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":2,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":3,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"},{"index":4,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":5,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":6,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":7,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":8,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":9,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%3:00:00::","synset_id":"new.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d001.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":860,"cached_input_tokens":0,"output_tokens":176,"reasoning_output_tokens":160},"cost":{"currency":"USD","total_usd":0.009580000000000002,"input_uncached_usd":0.0043,"input_cached_usd":0.0,"output_usd":0.00528,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8971210420131683}
{"item_id":"senseval2.d001.s018.t003","gold_sense_keys":["finding%1:17:00::"],"candidates":[{"index":1,"sense_key":"finding%1:04:00::","synset_id":"determination.n.01"},{"index":2,"sense_key":"finding%1:04:01::","synset_id":"finding.n.02"},{"index":3,"sense_key":"finding%1:17:00::","synset_id":"finding.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"finding%1:17:00::","call_ids":["senseval2.d001.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"finding%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":140},"cost":{"currency":"USD","total_usd":0.007145,"input_uncached_usd":0.002465,"input_cached_usd":0.0,"output_usd":0.00468,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.495717625017278}
{"item_id":"senseval2.d001.s019.t000","gold_sense_keys":["couple%1:14:01::"],"candidates":[{"index":1,"sense_key":"couple%1:17:00::","synset_id":"couple.n.05"},{"index":2,"sense_key":"couple%1:23:00::","synset_id":"couple.n.04"},{"index":3,"sense_key":"couple%1:23:01::","synset_id":"couple.n.03"},{"index":4,"sense_key":"couple%1:14:00::","synset_id":"couple.n.01"},{"index":5,"sense_key":"couple%1:14:01::","synset_id":"couple.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"couple%1:14:01::","call_ids":["senseval2.d001.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"couple%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.006315,"input_uncached_usd":0.0028350000000000003,"input_cached_usd":0.0,"output_usd":0.00348,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.846931125037372}
{"item_id":"senseval2.d001.s019.t001","gold_sense_keys":["know%2:31:01::","know%2:31:02::"],"candidates":[{"index":1,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":2,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":3,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":4,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":7,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":8,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":9,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":10,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d001.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":300,"reasoning_output_tokens":284},"cost":{"currency":"USD","total_usd":0.013495,"input_uncached_usd":0.004495000000000001,"input_cached_usd":0.0,"output_usd":0.009000000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.926495332969353}
{"item_id":"senseval2.d001.s019.t002","gold_sense_keys":["even%4:02:00::"],"candidates":[{"index":1,"sense_key":"even%4:02:00::","synset_id":"even.r.01"},{"index":2,"sense_key":"even%4:02:03::","synset_id":"even.r.02"},{"index":3,"sense_key":"even%4:02:01::","synset_id":"even.r.03"},{"index":4,"sense_key":"even%4:02:04::","synset_id":"even.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"even%4:02:00::","call_ids":["senseval2.d001.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"even%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":90},"cost":{"currency":"USD","total_usd":0.005895,"input_uncached_usd":0.002745,"input_cached_usd":0.0,"output_usd":0.00315,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.720463583013043}
{"item_id":"senseval2.d001.s019.t003","gold_sense_keys":["before%4:02:03::"],"candidates":[{"index":1,"sense_key":"before%4:02:01::","synset_id":"ahead.r.01"},{"index":2,"sense_key":"before%4:02:03::","synset_id":"earlier.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"before%4:02:03::","call_ids":["senseval2.d001.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"before%4:02:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":42,"reasoning_output_tokens":26},"cost":{"currency":"USD","total_usd":0.003615,"input_uncached_usd":0.0023550000000000003,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8265507500618696}
{"item_id":"senseval2.d001.s019.t004","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":2,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":3,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"become%2:30:00::","call_ids":["senseval2.d001.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"become%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.006280000000000001,"input_uncached_usd":0.0027400000000000002,"input_cached_usd":0.0,"output_usd":0.00354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.541894915979356}
{"item_id":"senseval2.d001.s019.t005","gold_sense_keys":["pregnant%3:00:00::"],"candidates":[{"index":1,"sense_key":"pregnant%5:00:00:full:00","synset_id":"fraught.s.02"},{"index":2,"sense_key":"pregnant%5:00:00:meaningful:00","synset_id":"meaning.s.01"},{"index":3,"sense_key":"pregnant%3:00:00::","synset_id":"pregnant.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"pregnant%3:00:00::","call_ids":["senseval2.d001.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"pregnant%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.00385,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7680788750294596}
{"item_id":"senseval2.d001.s019.t006","gold_sense_keys":["child%1:18:01::"],"candidates":[{"index":1,"sense_key":"child%1:18:07::","synset_id":"child.n.04"},{"index":2,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":3,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":4,"sense_key":"child%1:18:01::","synset_id":"child.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d001.s019.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"child%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":80},"cost":{"currency":"USD","total_usd":0.005665,"input_uncached_usd":0.002785,"input_cached_usd":0.0,"output_usd":0.00288,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5287989999633282}
{"item_id":"senseval2.d001.s019.t007","gold_sense_keys":["chance%1:07:00::"],"candidates":[{"index":1,"sense_key":"chance%1:26:00::","synset_id":"opportunity.n.01"},{"index":2,"sense_key":"chance%1:07:00::","synset_id":"probability.n.01"},{"index":3,"sense_key":"chance%1:26:01::","synset_id":"prospect.n.01"},{"index":4,"sense_key":"chance%1:04:00::","synset_id":"chance.n.03"},{"index":5,"sense_key":"chance%1:19:00::","synset_id":"luck.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"chance%1:07:00::","call_ids":["senseval2.d001.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"chance%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.004960000000000001,"input_uncached_usd":0.0031300000000000004,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5105114579200745}
{"item_id":"senseval2.d001.s019.t010","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"},{"index":2,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":3,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":4,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":5,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval2.d001.s019.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":580,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.00449,"input_uncached_usd":0.0029000000000000002,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.631186000071466}
{"item_id":"senseval2.d001.s019.t011","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":2,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":3,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s019.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.0043,"input_uncached_usd":0.0029200000000000003,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5652460829587653}
{"item_id":"senseval2.d001.s019.t012","gold_sense_keys":["occur%2:42:03::"],"candidates":[{"index":1,"sense_key":"occur%2:42:03::","synset_id":"occur.v.03"},{"index":2,"sense_key":"occur%2:30:00::","synset_id":"happen.v.01"},{"index":3,"sense_key":"occur%2:31:00::","synset_id":"occur.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"occur%2:42:03::","call_ids":["senseval2.d001.s019.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"occur%2:42:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":831,"reasoning_output_tokens":810},"cost":{"currency":"USD","total_usd":0.027565,"input_uncached_usd":0.002635,"input_cached_usd":0.0,"output_usd":0.02493,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":18.90250287496019}
{"item_id":"senseval2.d001.s019.t013","gold_sense_keys":["about%4:02:00::"],"candidates":[{"index":1,"sense_key":"about%4:02:04::","synset_id":"about.r.04"},{"index":2,"sense_key":"about%4:02:06::","synset_id":"about.r.02"},{"index":3,"sense_key":"about%4:02:02::","synset_id":"about.r.05"},{"index":4,"sense_key":"about%4:02:05::","synset_id":"about.r.07"},{"index":5,"sense_key":"about%4:02:03::","synset_id":"about.r.06"},{"index":6,"sense_key":"about%4:02:00::","synset_id":"approximately.r.01"},{"index":7,"sense_key":"about%4:02:01::","synset_id":"about.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"about%4:02:00::","call_ids":["senseval2.d001.s019.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"about%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":706,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.00542,"input_uncached_usd":0.00353,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.644619332975708}
{"item_id":"senseval2.d001.s019.t014","gold_sense_keys":["once%4:02:00::"],"candidates":[{"index":1,"sense_key":"once%4:02:00::","synset_id":"once.r.01"},{"index":2,"sense_key":"once%4:02:02::","synset_id":"once.r.02"},{"index":3,"sense_key":"once%4:02:01::","synset_id":"once.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"once%4:02:00::","call_ids":["senseval2.d001.s019.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"once%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.005165,"input_uncached_usd":0.0024950000000000003,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8440647079842165}
{"item_id":"senseval2.d001.s019.t015","gold_sense_keys":["birth%1:18:00::"],"candidates":[{"index":1,"sense_key":"birth%1:22:00::","synset_id":"parturition.n.01"},{"index":2,"sense_key":"birth%1:24:00::","synset_id":"parentage.n.02"},{"index":3,"sense_key":"birth%1:18:00::","synset_id":"birth.n.05"},{"index":4,"sense_key":"birth%1:28:00::","synset_id":"birth.n.01"},{"index":5,"sense_key":"birth%1:11:00::","synset_id":"birth.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"birth%1:18:00::","call_ids":["senseval2.d001.s019.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"birth%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":103},"cost":{"currency":"USD","total_usd":0.00645,"input_uncached_usd":0.0029100000000000003,"input_cached_usd":0.0,"output_usd":0.00354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6620549170766026}
{"item_id":"senseval2.d001.s020.t000","gold_sense_keys":["know%2:31:01::","know%2:31:02::"],"candidates":[{"index":1,"sense_key":"know%2:31:14::","synset_id":"know.v.11"},{"index":2,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":3,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":4,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":7,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":8,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":11,"sense_key":"know%2:31:12::","synset_id":"know.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d001.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":911,"cached_input_tokens":0,"output_tokens":373,"reasoning_output_tokens":357},"cost":{"currency":"USD","total_usd":0.015745000000000002,"input_uncached_usd":0.004555,"input_cached_usd":0.0,"output_usd":0.01119,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.243902083020657}
{"item_id":"senseval2.d001.s020.t001","gold_sense_keys":["carry%2:42:13::"],"candidates":[{"index":1,"sense_key":"carry%2:29:00::","synset_id":"have_a_bun_in_the_oven.v.01"},{"index":2,"sense_key":"carry%2:41:02::","synset_id":"carry.v.23"},{"index":3,"sense_key":"carry%2:33:01::","synset_id":"carry.v.11"},{"index":4,"sense_key":"carry%2:40:00::","synset_id":"stock.v.01"},{"index":5,"sense_key":"carry%2:33:02::","synset_id":"carry.v.37"},{"index":6,"sense_key":"carry%2:42:03::","synset_id":"hold.v.11"},{"index":7,"sense_key":"carry%2:41:00::","synset_id":"carry.v.08"},{"index":8,"sense_key":"carry%2:40:10::","synset_id":"carry.v.26"},{"index":9,"sense_key":"carry%2:34:00::","synset_id":"carry.v.33"},{"index":10,"sense_key":"carry%2:40:04::","synset_id":"carry.v.27"},{"index":11,"sense_key":"carry%2:36:00::","synset_id":"carry.v.15"},{"index":12,"sense_key":"carry%2:33:12::","synset_id":"carry.v.38"},{"index":13,"sense_key":"carry%2:35:09::","synset_id":"carry.v.05"},{"index":14,"sense_key":"carry%2:42:08::","synset_id":"carry.v.22"},{"index":15,"sense_key":"carry%2:40:13::","synset_id":"post.v.07"},{"index":16,"sense_key":"carry%2:41:12::","synset_id":"carry.v.25"},{"index":17,"sense_key":"carry%2:41:01::","synset_id":"behave.v.02"},{"index":18,"sense_key":"carry%2:38:03::","synset_id":"impart.v.03"},{"index":19,"sense_key":"carry%2:35:08::","synset_id":"dribble.v.03"},{"index":20,"sense_key":"carry%2:42:14::","synset_id":"carry.v.20"},{"index":21,"sense_key":"carry%2:32:00::","synset_id":"carry.v.39"},{"index":22,"sense_key":"carry%2:38:07::","synset_id":"carry.v.30"},{"index":23,"sense_key":"carry%2:42:12::","synset_id":"carry.v.21"},{"index":24,"sense_key":"carry%2:41:06::","synset_id":"carry.v.24"},{"index":25,"sense_key":"carry%2:35:12::","synset_id":"carry.v.32"},{"index":26,"sense_key":"carry%2:30:05::","synset_id":"carry.v.09"},{"index":27,"sense_key":"carry%2:32:07::","synset_id":"carry.v.17"},{"index":28,"sense_key":"carry%2:31:12::","synset_id":"carry.v.12"},{"index":29,"sense_key":"carry%2:42:02::","synset_id":"carry.v.10"},{"index":30,"sense_key":"carry%2:42:15::","synset_id":"carry.v.19"},{"index":31,"sense_key":"carry%2:33:03::","synset_id":"carry.v.36"},{"index":32,"sense_key":"carry%2:34:01::","synset_id":"carry.v.34"},{"index":33,"sense_key":"carry%2:35:01::","synset_id":"hold.v.14"},{"index":34,"sense_key":"carry%2:33:04::","synset_id":"carry.v.35"},{"index":35,"sense_key":"carry%2:36:08::","synset_id":"carry.v.31"},{"index":36,"sense_key":"carry%2:42:13::","synset_id":"carry.v.18"},{"index":37,"sense_key":"carry%2:42:01::","synset_id":"carry.v.02"},{"index":38,"sense_key":"carry%2:35:02::","synset_id":"transport.v.02"},{"index":39,"sense_key":"carry%2:32:02::","synset_id":"carry.v.04"},{"index":40,"sense_key":"carry%2:38:10::","synset_id":"carry.v.29"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"carry%2:42:03::","call_ids":["senseval2.d001.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"carry%2:42:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1985,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.025765000000000003,"input_uncached_usd":0.009925000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.023005625000224}
{"item_id":"senseval2.d001.s020.t002","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"},{"index":2,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.004065,"input_uncached_usd":0.002415,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5069961659610271}
{"item_id":"senseval2.d001.s020.t004","gold_sense_keys":["lose%2:40:02::"],"candidates":[{"index":1,"sense_key":"lose%2:40:00::","synset_id":"lose.v.08"},{"index":2,"sense_key":"lose%2:30:03::","synset_id":"suffer.v.11"},{"index":3,"sense_key":"lose%2:39:01::","synset_id":"miss.v.01"},{"index":4,"sense_key":"lose%2:40:02::","synset_id":"lose.v.01"},{"index":5,"sense_key":"lose%2:40:01::","synset_id":"lose.v.05"},{"index":6,"sense_key":"lose%2:33:00::","synset_id":"lose.v.02"},{"index":7,"sense_key":"lose%2:35:00::","synset_id":"misplace.v.01"},{"index":8,"sense_key":"lose%2:33:01::","synset_id":"fall_back.v.04"},{"index":9,"sense_key":"lose%2:37:00::","synset_id":"lose.v.03"},{"index":10,"sense_key":"lose%2:40:06::","synset_id":"lose.v.07"},{"index":11,"sense_key":"lose%2:39:00::","synset_id":"lose.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"lose%2:40:02::","call_ids":["senseval2.d001.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"lose%2:40:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":860,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":114},"cost":{"currency":"USD","total_usd":0.0082,"input_uncached_usd":0.0043,"input_cached_usd":0.0,"output_usd":0.0039000000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.121455082902685}
{"item_id":"senseval2.d001.s020.t005","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":2,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":3,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval2.d001.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00491,"input_uncached_usd":0.0029600000000000004,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0746132080676034}
{"item_id":"senseval2.d001.s020.t008","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":2,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":3,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":4,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":5,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":6,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"only%4:02:02::","call_ids":["senseval2.d001.s020.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"only%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":675,"cached_input_tokens":0,"output_tokens":179,"reasoning_output_tokens":163},"cost":{"currency":"USD","total_usd":0.008745000000000001,"input_uncached_usd":0.0033750000000000004,"input_cached_usd":0.0,"output_usd":0.00537,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9186002500355244}
{"item_id":"senseval2.d001.s020.t009","gold_sense_keys":["mother%1:18:00::"],"candidates":[{"index":1,"sense_key":"mother%1:18:02::","synset_id":"mother.n.03"},{"index":2,"sense_key":"mother%1:18:00::","synset_id":"mother.n.01"},{"index":3,"sense_key":"mother%1:27:00::","synset_id":"mother.n.02"},{"index":4,"sense_key":"mother%1:18:01::","synset_id":"mother.n.04"},{"index":5,"sense_key":"mother%1:09:00::","synset_id":"mother.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mother%1:18:00::","call_ids":["senseval2.d001.s020.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mother%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00466,"input_uncached_usd":0.00286,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6691527500515804}
{"item_id":"senseval2.d001.s020.t010","gold_sense_keys":["suffer%2:39:01::","suffer%2:39:02::"],"candidates":[{"index":1,"sense_key":"suffer%2:29:01::","synset_id":"suffer.v.02"},{"index":2,"sense_key":"suffer%2:30:00::","synset_id":"suffer.v.05"},{"index":3,"sense_key":"suffer%2:37:00::","synset_id":"suffer.v.03"},{"index":4,"sense_key":"suffer%2:31:00::","synset_id":"digest.v.03"},{"index":5,"sense_key":"suffer%2:30:02::","synset_id":"suffer.v.11"},{"index":6,"sense_key":"suffer%2:39:00::","synset_id":"hurt.v.06"},{"index":7,"sense_key":"suffer%2:39:01::","synset_id":"suffer.v.01"},{"index":8,"sense_key":"suffer%2:42:04::","synset_id":"suffer.v.09"},{"index":9,"sense_key":"suffer%2:29:00::","synset_id":"suffer.v.06"},{"index":10,"sense_key":"suffer%2:29:03::","synset_id":"suffer.v.08"},{"index":11,"sense_key":"suffer%2:39:02::","synset_id":"suffer.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"suffer%2:39:02::","call_ids":["senseval2.d001.s020.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"suffer%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":196,"reasoning_output_tokens":180},"cost":{"currency":"USD","total_usd":0.009925,"input_uncached_usd":0.004045,"input_cached_usd":0.0,"output_usd":0.00588,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.600004667066969}
{"item_id":"senseval2.d001.s020.t011","gold_sense_keys":["same%3:00:00::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"},{"index":4,"sense_key":"same%3:00:04::","synset_id":"like.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"same%3:00:00::","call_ids":["senseval2.d001.s020.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"same%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":1273,"reasoning_output_tokens":1245},"cost":{"currency":"USD","total_usd":0.040975000000000004,"input_uncached_usd":0.002785,"input_cached_usd":0.0,"output_usd":0.03819,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":27.878707083058544}
{"item_id":"senseval2.d001.s020.t012","gold_sense_keys":["fate%1:26:00::"],"candidates":[{"index":1,"sense_key":"fate%1:11:00::","synset_id":"destiny.n.01"},{"index":2,"sense_key":"fate%1:18:00::","synset_id":"destiny.n.02"},{"index":3,"sense_key":"fate%1:26:00::","synset_id":"fortune.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fate%1:26:00::","call_ids":["senseval2.d001.s020.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fate%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01854,"input_uncached_usd":0.0027300000000000002,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.897592999972403}
{"item_id":"senseval2.d001.s020.t013","gold_sense_keys":["baby%1:18:00::"],"candidates":[{"index":1,"sense_key":"baby%1:18:00::","synset_id":"baby.n.01"},{"index":2,"sense_key":"baby%1:18:03::","synset_id":"child.n.03"},{"index":3,"sense_key":"baby%1:18:01::","synset_id":"baby.n.02"},{"index":4,"sense_key":"baby%1:04:00::","synset_id":"baby.n.07"},{"index":5,"sense_key":"baby%1:18:02::","synset_id":"baby.n.05"},{"index":6,"sense_key":"baby%1:18:04::","synset_id":"baby.n.04"},{"index":7,"sense_key":"baby%1:05:00::","synset_id":"baby.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"baby%1:18:00::","call_ids":["senseval2.d001.s020.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"baby%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.005655,"input_uncached_usd":0.0034050000000000005,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.695401458069682}
{"item_id":"senseval2.d001.s021.t000","gold_sense_keys":["isolation%1:04:00::"],"candidates":[{"index":1,"sense_key":"isolation%1:04:01::","synset_id":"isolation.n.05"},{"index":2,"sense_key":"isolation%1:26:00::","synset_id":"isolation.n.01"},{"index":3,"sense_key":"isolation%1:22:00::","synset_id":"isolation.n.04"},{"index":4,"sense_key":"isolation%1:04:00::","synset_id":"isolation.n.03"},{"index":5,"sense_key":"isolation%1:12:00::","synset_id":"isolation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"isolation%1:04:00::","call_ids":["senseval2.d001.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"isolation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.005265000000000001,"input_uncached_usd":0.0028650000000000004,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0729546250076964}
{"item_id":"senseval2.d001.s021.t003","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":3,"sense_key":"become%2:42:03::","synset_id":"become.v.04"},{"index":4,"sense_key":"become%2:42:00::","synset_id":"become.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"become%2:30:00::","call_ids":["senseval2.d001.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"become%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":581,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":85},"cost":{"currency":"USD","total_usd":0.0059050000000000005,"input_uncached_usd":0.002905,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2778457920067012}
{"item_id":"senseval2.d001.s021.t004","gold_sense_keys":["possible%3:00:00::"],"candidates":[{"index":1,"sense_key":"possible%3:00:04::","synset_id":"potential.a.01"},{"index":2,"sense_key":"possible%3:00:00::","synset_id":"possible.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"possible%3:00:00::","call_ids":["senseval2.d001.s021.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"possible%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":110},"cost":{"currency":"USD","total_usd":0.006215,"input_uncached_usd":0.002435,"input_cached_usd":0.0,"output_usd":0.00378,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.365393290994689}
{"item_id":"senseval2.d001.s021.t005","gold_sense_keys":["last%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"last%5:00:00:ultimate:00","synset_id":"final.s.02"},{"index":2,"sense_key":"last%5:00:00:high:02","synset_id":"last.s.06"},{"index":3,"sense_key":"last%5:00:00:unlikely:00","synset_id":"last.s.03"},{"index":4,"sense_key":"last%5:00:00:closing:00","synset_id":"concluding.s.01"},{"index":5,"sense_key":"last%3:00:00::","synset_id":"last.a.02"},{"index":6,"sense_key":"last%5:00:00:dying:00","synset_id":"last.s.04"},{"index":7,"sense_key":"last%5:00:00:unalterable:00","synset_id":"final.s.03"},{"index":8,"sense_key":"last%5:00:00:worst:00","synset_id":"last.s.08"},{"index":9,"sense_key":"last%5:00:00:past:00","synset_id":"last.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"last%5:00:00:past:00","call_ids":["senseval2.d001.s021.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"last%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":829,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.006215,"input_uncached_usd":0.004145,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8404299579560757}
{"item_id":"senseval2.d001.s021.t007","gold_sense_keys":["find_out%2:32:00::"],"candidates":[{"index":1,"sense_key":"find_out%2:31:00::","synset_id":"learn.v.02"},{"index":2,"sense_key":"find_out%2:32:00::","synset_id":"determine.v.01"},{"index":3,"sense_key":"find_out%2:31:01::","synset_id":"find_out.v.04"},{"index":4,"sense_key":"find_out%2:32:01::","synset_id":"determine.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"find_out%2:32:01::","call_ids":["senseval2.d001.s021.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"find_out%2:32:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019155,"input_uncached_usd":0.0033150000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.072104041930288}
{"item_id":"senseval2.d001.s021.t008","gold_sense_keys":["threat%1:26:00::"],"candidates":[{"index":1,"sense_key":"threat%1:18:00::","synset_id":"terror.n.02"},{"index":2,"sense_key":"threat%1:26:00::","synset_id":"menace.n.01"},{"index":3,"sense_key":"threat%1:10:00::","synset_id":"threat.n.02"},{"index":4,"sense_key":"threat%1:10:01::","synset_id":"threat.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"threat%1:26:00::","call_ids":["senseval2.d001.s021.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"threat%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":576,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.005220000000000001,"input_uncached_usd":0.00288,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0952568750362843}
{"item_id":"senseval2.d001.s021.t009","gold_sense_keys":["baby%1:18:00::"],"candidates":[{"index":1,"sense_key":"baby%1:05:00::","synset_id":"baby.n.06"},{"index":2,"sense_key":"baby%1:18:02::","synset_id":"baby.n.05"},{"index":3,"sense_key":"baby%1:18:03::","synset_id":"child.n.03"},{"index":4,"sense_key":"baby%1:18:01::","synset_id":"baby.n.02"},{"index":5,"sense_key":"baby%1:04:00::","synset_id":"baby.n.07"},{"index":6,"sense_key":"baby%1:18:00::","synset_id":"baby.n.01"},{"index":7,"sense_key":"baby%1:18:04::","synset_id":"baby.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"baby%1:18:00::","call_ids":["senseval2.d001.s021.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"baby%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":703,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019355,"input_uncached_usd":0.0035150000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.975726083037443}
{"item_id":"senseval2.d001.s021.t010","gold_sense_keys":["face%2:32:00::"],"candidates":[{"index":1,"sense_key":"face%2:35:01::","synset_id":"face.v.09"},{"index":2,"sense_key":"face%2:32:00::","synset_id":"confront.v.02"},{"index":3,"sense_key":"face%2:33:00::","synset_id":"confront.v.01"},{"index":4,"sense_key":"face%2:42:02::","synset_id":"face.v.04"},{"index":5,"sense_key":"face%2:39:04::","synset_id":"face.v.07"},{"index":6,"sense_key":"face%2:32:03::","synset_id":"confront.v.03"},{"index":7,"sense_key":"face%2:38:00::","synset_id":"face.v.05"},{"index":8,"sense_key":"face%2:42:00::","synset_id":"front.v.01"},{"index":9,"sense_key":"face%2:35:00::","synset_id":"face.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"face%2:32:00::","call_ids":["senseval2.d001.s021.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"face%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":799,"cached_input_tokens":0,"output_tokens":250,"reasoning_output_tokens":234},"cost":{"currency":"USD","total_usd":0.011495000000000002,"input_uncached_usd":0.003995,"input_cached_usd":0.0,"output_usd":0.007500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.971203790977597}
{"item_id":"senseval2.d001.s022.t001","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":2,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":3,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":4,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":5,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":6,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d001.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.005205,"input_uncached_usd":0.0033450000000000003,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6060524999629706}
{"item_id":"senseval2.d001.s022.t002","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":2,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":3,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":4,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":5,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":6,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"},{"index":7,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":8,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":9,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":10,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":11,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d001.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":871,"cached_input_tokens":0,"output_tokens":217,"reasoning_output_tokens":201},"cost":{"currency":"USD","total_usd":0.010865,"input_uncached_usd":0.004355,"input_cached_usd":0.0,"output_usd":0.00651,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.212521667010151}
{"item_id":"senseval2.d001.s022.t003","gold_sense_keys":["genetic%3:01:00::","genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":2,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"},{"index":3,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":4,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":564,"cached_input_tokens":0,"output_tokens":210,"reasoning_output_tokens":195},"cost":{"currency":"USD","total_usd":0.00912,"input_uncached_usd":0.00282,"input_cached_usd":0.0,"output_usd":0.0063,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.076934791984968}
{"item_id":"senseval2.d001.s022.t004","gold_sense_keys":["show%2:31:00::","show%2:32:00::"],"candidates":[{"index":1,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":2,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":3,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":4,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":5,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":6,"sense_key":"show%2:33:00::","synset_id":"show.v.12"},{"index":7,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":8,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":9,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":10,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":11,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":12,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"show%2:32:00::","call_ids":["senseval2.d001.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"show%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":956,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":137},"cost":{"currency":"USD","total_usd":0.00937,"input_uncached_usd":0.00478,"input_cached_usd":0.0,"output_usd":0.00459,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4110641659935936}
{"item_id":"senseval2.d001.s022.t005","gold_sense_keys":["little%5:00:00:emotional:00","little%5:00:00:young:00"],"candidates":[{"index":1,"sense_key":"little%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":2,"sense_key":"little%5:00:00:unimportant:00","synset_id":"fiddling.s.01"},{"index":3,"sense_key":"little%5:00:00:soft:04","synset_id":"little.s.03"},{"index":4,"sense_key":"little%3:00:00::","synset_id":"short.a.03"},{"index":5,"sense_key":"little%3:00:03::","synset_id":"little.a.02"},{"index":6,"sense_key":"little%5:00:00:young:00","synset_id":"little.s.01"},{"index":7,"sense_key":"little%3:00:01::","synset_id":"small.a.01"},{"index":8,"sense_key":"little%5:00:00:emotional:00","synset_id":"little.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"little%5:00:00:young:00","call_ids":["senseval2.d001.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"little%5:00:00:young:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":772,"cached_input_tokens":0,"output_tokens":209,"reasoning_output_tokens":193},"cost":{"currency":"USD","total_usd":0.01013,"input_uncached_usd":0.00386,"input_cached_usd":0.0,"output_usd":0.00627,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.440890291007236}
{"item_id":"senseval2.d001.s022.t006","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"},{"index":2,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"},{"index":3,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.003885,"input_uncached_usd":0.002445,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":50.702005417086184}
{"item_id":"senseval2.d001.s022.t007","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"},{"index":2,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s022.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004385,"input_uncached_usd":0.0023750000000000004,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.02520725002978}
{"item_id":"senseval2.d001.s022.t011","gold_sense_keys":["face%2:32:00::"],"candidates":[{"index":1,"sense_key":"face%2:39:04::","synset_id":"face.v.07"},{"index":2,"sense_key":"face%2:38:00::","synset_id":"face.v.05"},{"index":3,"sense_key":"face%2:42:02::","synset_id":"face.v.04"},{"index":4,"sense_key":"face%2:32:00::","synset_id":"confront.v.02"},{"index":5,"sense_key":"face%2:42:00::","synset_id":"front.v.01"},{"index":6,"sense_key":"face%2:35:01::","synset_id":"face.v.09"},{"index":7,"sense_key":"face%2:32:03::","synset_id":"confront.v.03"},{"index":8,"sense_key":"face%2:35:00::","synset_id":"face.v.08"},{"index":9,"sense_key":"face%2:33:00::","synset_id":"confront.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"face%2:32:00::","call_ids":["senseval2.d001.s022.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"face%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":167,"reasoning_output_tokens":151},"cost":{"currency":"USD","total_usd":0.008865000000000001,"input_uncached_usd":0.0038550000000000004,"input_cached_usd":0.0,"output_usd":0.00501,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.569215999916196}
{"item_id":"senseval2.d001.s022.t012","gold_sense_keys":["more%3:00:01::"],"candidates":[{"index":1,"sense_key":"more%3:00:02::","synset_id":"more.a.02"},{"index":2,"sense_key":"more%3:00:01::","synset_id":"more.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"more%3:00:01::","call_ids":["senseval2.d001.s022.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"more%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.00538,"input_uncached_usd":0.0024400000000000003,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.332926500006579}
{"item_id":"senseval2.d001.s022.t013","gold_sense_keys":["risk%1:26:00::"],"candidates":[{"index":1,"sense_key":"risk%1:26:00::","synset_id":"hazard.n.01"},{"index":2,"sense_key":"risk%1:04:00::","synset_id":"risk.n.02"},{"index":3,"sense_key":"risk%1:07:02::","synset_id":"risk.n.03"},{"index":4,"sense_key":"risk%1:07:01::","synset_id":"risk.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"risk%1:26:00::","call_ids":["senseval2.d001.s022.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"risk%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":123},"cost":{"currency":"USD","total_usd":0.006975,"input_uncached_usd":0.0028350000000000003,"input_cached_usd":0.0,"output_usd":0.0041400000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.345820124959573}
{"item_id":"senseval2.d001.s022.t014","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":2,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":3,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":4,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s022.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":561,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004845,"input_uncached_usd":0.002805,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.5394943749997765}
{"item_id":"senseval2.d001.s022.t015","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":2,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":3,"sense_key":"child%1:18:07::","synset_id":"child.n.04"},{"index":4,"sense_key":"child%1:18:01::","synset_id":"child.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"child%1:18:00::","call_ids":["senseval2.d001.s022.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"child%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":561,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.006345,"input_uncached_usd":0.002805,"input_cached_usd":0.0,"output_usd":0.00354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.643807707936503}
{"item_id":"senseval2.d001.s022.t016","gold_sense_keys":["develop%2:29:00::"],"candidates":[{"index":1,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":2,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":3,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":6,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":7,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":8,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":9,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":10,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":11,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":12,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":13,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":14,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":15,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":16,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":17,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":20,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":21,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"develop%2:29:00::","call_ids":["senseval2.d001.s022.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"develop%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1329,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.009225,"input_uncached_usd":0.006645000000000001,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4097803339827806}
{"item_id":"senseval2.d001.s022.t018","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":3,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s022.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.004495000000000001,"input_uncached_usd":0.002935,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3149192499695346}
{"item_id":"senseval2.d001.s023.t000","gold_sense_keys":["make%2:30:10::"],"candidates":[{"index":1,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":2,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":3,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":4,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":5,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":6,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":7,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":8,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":9,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":10,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":11,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":12,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":13,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":14,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":15,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":16,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":17,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":18,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":19,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":20,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":21,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":22,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":23,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":24,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":25,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":26,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":27,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":28,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":29,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":30,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":31,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":32,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":33,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":34,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":35,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":36,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":37,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":38,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":39,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":40,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":41,"sense_key":"make%2:29:01::","synset_id":"make.v.49"},{"index":42,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":43,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":44,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":45,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":46,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":47,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":48,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":49,"sense_key":"make%2:36:00::","synset_id":"make.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"make%2:30:10::","call_ids":["senseval2.d001.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"make%2:30:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2328,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":79},"cost":{"currency":"USD","total_usd":0.014490000000000001,"input_uncached_usd":0.011640000000000001,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.325821041013114}
{"item_id":"senseval2.d001.s023.t002","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":2,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":5,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":6,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":7,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":8,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":9,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":10,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":11,"sense_key":"say%2:32:15::","synset_id":"say.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d001.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":371,"reasoning_output_tokens":355},"cost":{"currency":"USD","total_usd":0.015390000000000001,"input_uncached_usd":0.004260000000000001,"input_cached_usd":0.0,"output_usd":0.011130000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.778841041959822}
{"item_id":"senseval2.d001.s024.t001","gold_sense_keys":["first%3:00:00::"],"candidates":[{"index":1,"sense_key":"first%3:00:00::","synset_id":"first.a.01"},{"index":2,"sense_key":"first%5:00:02:opening:00","synset_id":"inaugural.s.01"},{"index":3,"sense_key":"first%3:00:03::","synset_id":"first.a.06"},{"index":4,"sense_key":"first%5:00:00:best:00","synset_id":"first.s.04"},{"index":5,"sense_key":"first%5:00:00:ordinal:00","synset_id":"first.s.01"},{"index":6,"sense_key":"first%5:00:01:opening:00","synset_id":"beginning.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"first%3:00:00::","call_ids":["senseval2.d001.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"first%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":364,"reasoning_output_tokens":348},"cost":{"currency":"USD","total_usd":0.01421,"input_uncached_usd":0.0032900000000000004,"input_cached_usd":0.0,"output_usd":0.010920000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.778701083967462}
{"item_id":"senseval2.d001.s024.t004","gold_sense_keys":["individual%1:03:00::"],"candidates":[{"index":1,"sense_key":"individual%1:03:00::","synset_id":"person.n.01"},{"index":2,"sense_key":"individual%1:18:00::","synset_id":"individual.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"individual%1:03:00::","call_ids":["senseval2.d001.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"individual%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":97},"cost":{"currency":"USD","total_usd":0.005665,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.00336,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.599238707916811}
{"item_id":"senseval2.d001.s024.t005","gold_sense_keys":["expect%2:31:00::"],"candidates":[{"index":1,"sense_key":"expect%2:29:02::","synset_id":"expect.v.05"},{"index":2,"sense_key":"expect%2:31:05::","synset_id":"expect.v.04"},{"index":3,"sense_key":"expect%2:29:00::","synset_id":"have_a_bun_in_the_oven.v.01"},{"index":4,"sense_key":"expect%2:32:00::","synset_id":"ask.v.04"},{"index":5,"sense_key":"expect%2:31:01::","synset_id":"expect.v.03"},{"index":6,"sense_key":"expect%2:31:00::","synset_id":"expect.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"expect%2:31:00::","call_ids":["senseval2.d001.s024.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"expect%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":660,"cached_input_tokens":0,"output_tokens":185,"reasoning_output_tokens":169},"cost":{"currency":"USD","total_usd":0.00885,"input_uncached_usd":0.0033000000000000004,"input_cached_usd":0.0,"output_usd":0.00555,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.404993791948073}
{"item_id":"senseval2.d001.s024.t006","gold_sense_keys":["develop%2:29:00::"],"candidates":[{"index":1,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":2,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":3,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":4,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":5,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":8,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":9,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":10,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":11,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":12,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":13,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":14,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":15,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":16,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":17,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":18,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":19,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"},{"index":20,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":21,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"develop%2:29:00::","call_ids":["senseval2.d001.s024.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"develop%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1340,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.00964,"input_uncached_usd":0.0067,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.178494874970056}
{"item_id":"senseval2.d001.s024.t007","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":2,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":3,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":4,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":5,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.004345,"input_uncached_usd":0.0029950000000000003,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5276774170342833}
{"item_id":"senseval2.d001.s025.t000","gold_sense_keys":["equally%4:02:01::"],"candidates":[{"index":1,"sense_key":"equally%4:02:02::","synset_id":"evenly.r.01"},{"index":2,"sense_key":"equally%4:02:01::","synset_id":"equally.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"equally%4:02:01::","call_ids":["senseval2.d001.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"equally%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.004580000000000001,"input_uncached_usd":0.0024500000000000004,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0920302500016987}
{"item_id":"senseval2.d001.s025.t001","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"},{"index":2,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":3,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"},{"index":4,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"},{"index":5,"sense_key":"important%3:00:00::","synset_id":"important.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"important%3:00:00::","call_ids":["senseval2.d001.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"important%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018885,"input_uncached_usd":0.0030450000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":492.5390353749972}
{"item_id":"senseval2.d001.s025.t003","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"},{"index":2,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":3,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":4,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"discovery%1:09:00::","call_ids":["senseval2.d001.s025.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"discovery%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018415,"input_uncached_usd":0.002575,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.11077137500979}
{"item_id":"senseval2.d001.s025.t005","gold_sense_keys":["control%2:41:01::"],"candidates":[{"index":1,"sense_key":"control%2:41:02::","synset_id":"control.v.05"},{"index":2,"sense_key":"control%2:31:00::","synset_id":"master.v.04"},{"index":3,"sense_key":"control%2:35:00::","synset_id":"operate.v.03"},{"index":4,"sense_key":"control%2:41:01::","synset_id":"control.v.02"},{"index":5,"sense_key":"control%2:41:00::","synset_id":"control.v.01"},{"index":6,"sense_key":"control%2:37:06::","synset_id":"manipulate.v.05"},{"index":7,"sense_key":"control%2:31:01::","synset_id":"see.v.10"},{"index":8,"sense_key":"control%2:31:11::","synset_id":"control.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"control%2:41:01::","call_ids":["senseval2.d001.s025.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"control%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":772,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":133},"cost":{"currency":"USD","total_usd":0.0083,"input_uncached_usd":0.00386,"input_cached_usd":0.0,"output_usd":0.00444,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.165018875035457}
{"item_id":"senseval2.d001.s025.t007","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":3,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":4,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":5,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":6,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":7,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s025.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.00476,"input_uncached_usd":0.0032900000000000004,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.890759167028591}
{"item_id":"senseval2.d001.s025.t008","gold_sense_keys":["growth%1:22:00::"],"candidates":[{"index":1,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":2,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":3,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":4,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":5,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":6,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":7,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"growth%1:22:02::","call_ids":["senseval2.d001.s025.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"growth%1:22:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":389,"reasoning_output_tokens":373},"cost":{"currency":"USD","total_usd":0.015055,"input_uncached_usd":0.0033850000000000004,"input_cached_usd":0.0,"output_usd":0.01167,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.339849167037755}
{"item_id":"senseval2.d001.s025.t009","gold_sense_keys":["make%2:36:12::"],"candidates":[{"index":1,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":2,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":3,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":4,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":5,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":6,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":7,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":8,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":9,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":10,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":11,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":12,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":13,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":14,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":15,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":16,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":17,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":18,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":19,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":20,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":21,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":22,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":23,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":24,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":25,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":26,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":27,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":28,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":29,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":30,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":31,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":32,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":33,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":34,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":35,"sense_key":"make%2:29:01::","synset_id":"make.v.49"},{"index":36,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":37,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":38,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":39,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":40,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":41,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":44,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":45,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":46,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":47,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":48,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":49,"sense_key":"make%2:30:14::","synset_id":"make.v.46"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"make%2:36:12::","call_ids":["senseval2.d001.s025.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"make%2:36:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2342,"cached_input_tokens":0,"output_tokens":1564,"reasoning_output_tokens":1537},"cost":{"currency":"USD","total_usd":0.05863,"input_uncached_usd":0.011710000000000002,"input_cached_usd":0.0,"output_usd":0.04692,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":33.426178666064516}
{"item_id":"senseval2.d001.s025.t010","gold_sense_keys":["doctor%1:18:00::"],"candidates":[{"index":1,"sense_key":"doctor%1:18:00::","synset_id":"doctor.n.01"},{"index":2,"sense_key":"doctor%1:04:00::","synset_id":"doctor.n.03"},{"index":3,"sense_key":"doctor%1:18:02::","synset_id":"doctor_of_the_church.n.01"},{"index":4,"sense_key":"doctor%1:18:01::","synset_id":"doctor.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"doctor%1:18:00::","call_ids":["senseval2.d001.s025.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"doctor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":205,"reasoning_output_tokens":189},"cost":{"currency":"USD","total_usd":0.009130000000000001,"input_uncached_usd":0.0029800000000000004,"input_cached_usd":0.0,"output_usd":0.00615,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.619518749997951}
{"item_id":"senseval2.d001.s025.t011","gold_sense_keys":["name%2:32:03::"],"candidates":[{"index":1,"sense_key":"name%2:41:01::","synset_id":"name.v.03"},{"index":2,"sense_key":"name%2:41:00::","synset_id":"appoint.v.01"},{"index":3,"sense_key":"name%2:32:04::","synset_id":"mention.v.01"},{"index":4,"sense_key":"name%2:32:07::","synset_id":"name.v.05"},{"index":5,"sense_key":"name%2:32:03::","synset_id":"name.v.01"},{"index":6,"sense_key":"name%2:32:09::","synset_id":"list.v.01"},{"index":7,"sense_key":"name%2:31:00::","synset_id":"diagnose.v.01"},{"index":8,"sense_key":"name%2:31:01::","synset_id":"identify.v.05"},{"index":9,"sense_key":"name%2:32:02::","synset_id":"name.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"name%2:32:03::","call_ids":["senseval2.d001.s025.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"name%2:32:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01971,"input_uncached_usd":0.00387,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.009420083020814}
{"item_id":"senseval2.d001.s025.t012","gold_sense_keys":["open%2:30:00::"],"candidates":[{"index":1,"sense_key":"open%2:30:01::","synset_id":"open.v.07"},{"index":2,"sense_key":"open%2:35:08::","synset_id":"unfold.v.04"},{"index":3,"sense_key":"open%2:33:00::","synset_id":"open.v.09"},{"index":4,"sense_key":"open%2:42:00::","synset_id":"open.v.08"},{"index":5,"sense_key":"open%2:40:04::","synset_id":"afford.v.04"},{"index":6,"sense_key":"open%2:30:00::","synset_id":"open.v.06"},{"index":7,"sense_key":"open%2:35:06::","synset_id":"open.v.03"},{"index":8,"sense_key":"open%2:35:00::","synset_id":"open.v.01"},{"index":9,"sense_key":"open%2:41:00::","synset_id":"open.v.02"},{"index":10,"sense_key":"open%2:41:01::","synset_id":"open.v.04"},{"index":11,"sense_key":"open%2:39:10::","synset_id":"open.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"open%2:30:00::","call_ids":["senseval2.d001.s025.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"open%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":95},"cost":{"currency":"USD","total_usd":0.007180000000000001,"input_uncached_usd":0.00385,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1397792500210926}
{"item_id":"senseval2.d001.s025.t014","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":2,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":5,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s025.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00461,"input_uncached_usd":0.0029300000000000003,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.651913500041701}
{"item_id":"senseval2.d001.s025.t016","gold_sense_keys":["recent%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"recent%5:00:00:new:00","synset_id":"recent.s.01"},{"index":2,"sense_key":"recent%5:00:00:past:00","synset_id":"late.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"recent%5:00:00:past:00","call_ids":["senseval2.d001.s025.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"recent%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.0040750000000000005,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5307293330552056}
{"item_id":"senseval2.d001.s025.t017","gold_sense_keys":["month%1:28:01::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["senseval2.d001.s025.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":247,"reasoning_output_tokens":232},"cost":{"currency":"USD","total_usd":0.009715,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.00741,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.965907625039108}
{"item_id":"senseval2.d001.s025.t018","gold_sense_keys":["explode%2:30:02::"],"candidates":[{"index":1,"sense_key":"explode%2:32:02::","synset_id":"explode.v.06"},{"index":2,"sense_key":"explode%2:35:00::","synset_id":"explode.v.05"},{"index":3,"sense_key":"explode%2:30:00::","synset_id":"explode.v.02"},{"index":4,"sense_key":"explode%2:32:00::","synset_id":"explode.v.07"},{"index":5,"sense_key":"explode%2:30:02::","synset_id":"explode.v.10"},{"index":6,"sense_key":"explode%2:31:01::","synset_id":"explode.v.08"},{"index":7,"sense_key":"explode%2:30:01::","synset_id":"explode.v.01"},{"index":8,"sense_key":"explode%2:31:00::","synset_id":"explode.v.03"},{"index":9,"sense_key":"explode%2:30:03::","synset_id":"explode.v.04"},{"index":10,"sense_key":"explode%2:30:04::","synset_id":"detonate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"explode%2:30:02::","call_ids":["senseval2.d001.s025.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"explode%2:30:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":768,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.006180000000000001,"input_uncached_usd":0.0038400000000000005,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.857839958043769}
{"item_id":"senseval2.d001.s026.t000","gold_sense_keys":["turn_out%2:42:00::","turn_out%2:42:01::"],"candidates":[{"index":1,"sense_key":"turn_out%2:30:00::","synset_id":"turn_out.v.05"},{"index":2,"sense_key":"turn_out%2:40:03::","synset_id":"turn_out.v.09"},{"index":3,"sense_key":"turn_out%2:42:02::","synset_id":"turn_out.v.04"},{"index":4,"sense_key":"turn_out%2:36:00::","synset_id":"bear.v.05"},{"index":5,"sense_key":"turn_out%2:41:03::","synset_id":"turn_out.v.08"},{"index":6,"sense_key":"turn_out%2:35:00::","synset_id":"eject.v.01"},{"index":7,"sense_key":"turn_out%2:35:01::","synset_id":"switch_off.v.01"},{"index":8,"sense_key":"turn_out%2:42:00::","synset_id":"prove.v.01"},{"index":9,"sense_key":"turn_out%2:38:00::","synset_id":"turn_out.v.10"},{"index":10,"sense_key":"turn_out%2:36:01::","synset_id":"turn_out.v.03"},{"index":11,"sense_key":"turn_out%2:42:01::","synset_id":"turn_out.v.02"},{"index":12,"sense_key":"turn_out%2:29:00::","synset_id":"get_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"turn_out%2:42:01::","call_ids":["senseval2.d001.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"turn_out%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":895,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":107},"cost":{"currency":"USD","total_usd":0.008165,"input_uncached_usd":0.004475000000000001,"input_cached_usd":0.0,"output_usd":0.00369,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8676949579967186}
{"item_id":"senseval2.d001.s026.t001","gold_sense_keys":["study%2:31:02::"],"candidates":[{"index":1,"sense_key":"study%2:31:02::","synset_id":"analyze.v.01"},{"index":2,"sense_key":"study%2:31:04::","synset_id":"study.v.06"},{"index":3,"sense_key":"study%2:31:01::","synset_id":"study.v.05"},{"index":4,"sense_key":"study%2:39:00::","synset_id":"study.v.03"},{"index":5,"sense_key":"study%2:31:03::","synset_id":"study.v.02"},{"index":6,"sense_key":"study%2:31:00::","synset_id":"learn.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%2:31:02::","call_ids":["senseval2.d001.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":104},"cost":{"currency":"USD","total_usd":0.006815,"input_uncached_usd":0.0032150000000000004,"input_cached_usd":0.0,"output_usd":0.0036,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.003046667086892}
{"item_id":"senseval2.d001.s026.t002","gold_sense_keys":["tragic%5:00:00:sad:00"],"candidates":[{"index":1,"sense_key":"tragic%5:00:00:sad:00","synset_id":"tragic.s.01"},{"index":2,"sense_key":"tragic%3:01:00::","synset_id":"tragic.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tragic%5:00:00:sad:00","call_ids":["senseval2.d001.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tragic%5:00:00:sad:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.00495,"input_uncached_usd":0.0022500000000000003,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.533773292088881}
{"item_id":"senseval2.d001.s026.t003","gold_sense_keys":["uncommon%3:00:00::"],"candidates":[{"index":1,"sense_key":"uncommon%5:00:00:extraordinary:00","synset_id":"rare.s.05"},{"index":2,"sense_key":"uncommon%3:00:00::","synset_id":"uncommon.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"uncommon%3:00:00::","call_ids":["senseval2.d001.s026.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"uncommon%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.004735,"input_uncached_usd":0.002425,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0565405410015956}
{"item_id":"senseval2.d001.s026.t005","gold_sense_keys":["make%2:30:00::"],"candidates":[{"index":1,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":2,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":3,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":4,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":5,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":6,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":7,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":8,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":9,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":10,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":11,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":12,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":13,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":15,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":16,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":17,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":18,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":19,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":22,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":23,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":24,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":25,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":26,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":27,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":28,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":29,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":30,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":31,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":32,"sense_key":"make%2:29:01::","synset_id":"make.v.49"},{"index":33,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":34,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":35,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":36,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":37,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":38,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":39,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":40,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":43,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":44,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":45,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":46,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":47,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":48,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":49,"sense_key":"make%2:30:12::","synset_id":"make.v.47"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":36,"chosen_sense_key":"make%2:30:00::","call_ids":["senseval2.d001.s026.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":36,"predicted_sense_key":"make%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2329,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.027455,"input_uncached_usd":0.011645,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.638811749988236}
{"item_id":"senseval2.d001.s026.t006","gold_sense_keys":["possible%3:00:00::"],"candidates":[{"index":1,"sense_key":"possible%3:00:00::","synset_id":"possible.a.01"},{"index":2,"sense_key":"possible%3:00:04::","synset_id":"potential.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"possible%3:00:00::","call_ids":["senseval2.d001.s026.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"possible%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":113},"cost":{"currency":"USD","total_usd":0.0060550000000000005,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.00384,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.495633832993917}
{"item_id":"senseval2.d001.s026.t008","gold_sense_keys":["insight%1:09:01::","insight%1:09:02::"],"candidates":[{"index":1,"sense_key":"insight%1:09:00::","synset_id":"insight.n.04"},{"index":2,"sense_key":"insight%1:09:02::","synset_id":"penetration.n.02"},{"index":3,"sense_key":"insight%1:12:00::","synset_id":"insight.n.02"},{"index":4,"sense_key":"insight%1:09:01::","synset_id":"insight.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"insight%1:09:02::","call_ids":["senseval2.d001.s026.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"insight%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":191,"reasoning_output_tokens":175},"cost":{"currency":"USD","total_usd":0.008255000000000002,"input_uncached_usd":0.0025250000000000003,"input_cached_usd":0.0,"output_usd":0.00573,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.0741866250755265}
{"item_id":"senseval2.d001.s026.t009","gold_sense_keys":["basic%3:00:00::"],"candidates":[{"index":1,"sense_key":"basic%5:00:00:standard:01","synset_id":"basic.s.01"},{"index":2,"sense_key":"basic%5:00:02:first:00","synset_id":"basic.s.02"},{"index":3,"sense_key":"basic%3:00:00::","synset_id":"basic.a.01"},{"index":4,"sense_key":"basic%5:00:00:alkaline:00","synset_id":"basic.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"basic%3:00:00::","call_ids":["senseval2.d001.s026.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"basic%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":541,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.0052250000000000005,"input_uncached_usd":0.0027050000000000004,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4040877079823986}
{"item_id":"senseval2.d001.s026.t010","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":2,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":3,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":4,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":5,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s026.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":43,"reasoning_output_tokens":27},"cost":{"currency":"USD","total_usd":0.00415,"input_uncached_usd":0.00286,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6089485000120476}
{"item_id":"senseval2.d001.s026.t011","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":2,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":3,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":4,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":5,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":6,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":7,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":8,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":9,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":10,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":11,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d001.s026.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":2593,"reasoning_output_tokens":2554},"cost":{"currency":"USD","total_usd":0.08205500000000002,"input_uncached_usd":0.0042650000000000006,"input_cached_usd":0.0,"output_usd":0.07779,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":40.97755662491545}
{"item_id":"senseval2.d001.s026.t012","gold_sense_keys":["director%1:18:00::"],"candidates":[{"index":1,"sense_key":"director%1:18:00::","synset_id":"director.n.01"},{"index":2,"sense_key":"director%1:18:04::","synset_id":"film_director.n.01"},{"index":3,"sense_key":"director%1:18:03::","synset_id":"director.n.02"},{"index":4,"sense_key":"director%1:18:02::","synset_id":"director.n.03"},{"index":5,"sense_key":"director%1:18:01::","synset_id":"conductor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"director%1:18:00::","call_ids":["senseval2.d001.s026.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"director%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.00475,"input_uncached_usd":0.00265,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.992348625091836}
{"item_id":"senseval2.d001.s027.t001","gold_sense_keys":["public%1:14:00::"],"candidates":[{"index":1,"sense_key":"public%1:14:01::","synset_id":"public.n.02"},{"index":2,"sense_key":"public%1:14:00::","synset_id":"populace.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"public%1:14:00::","call_ids":["senseval2.d001.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"public%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":429,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.004635,"input_uncached_usd":0.002145,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9763619169825688}
{"item_id":"senseval2.d001.s027.t002","gold_sense_keys":["concerned%3:00:00::"],"candidates":[{"index":1,"sense_key":"concerned%5:00:00:involved:00","synset_id":"concerned.s.01"},{"index":2,"sense_key":"concerned%5:00:01:involved:00","synset_id":"implicated.s.01"},{"index":3,"sense_key":"concerned%3:00:00::","synset_id":"concerned.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"concerned%5:00:00:involved:00","call_ids":["senseval2.d001.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"concerned%5:00:00:involved:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":287,"reasoning_output_tokens":271},"cost":{"currency":"USD","total_usd":0.011165,"input_uncached_usd":0.0025550000000000004,"input_cached_usd":0.0,"output_usd":0.00861,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.486176540958695}
{"item_id":"senseval2.d001.s027.t003","gold_sense_keys":["advance%1:11:01::"],"candidates":[{"index":1,"sense_key":"advance%1:04:00::","synset_id":"progress.n.02"},{"index":2,"sense_key":"advance%1:10:00::","synset_id":"overture.n.03"},{"index":3,"sense_key":"advance%1:04:01::","synset_id":"advance.n.06"},{"index":4,"sense_key":"advance%1:21:00::","synset_id":"advance.n.05"},{"index":5,"sense_key":"advance%1:11:00::","synset_id":"progress.n.03"},{"index":6,"sense_key":"advance%1:11:01::","synset_id":"improvement.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"advance%1:11:01::","call_ids":["senseval2.d001.s027.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"advance%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":580,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":109},"cost":{"currency":"USD","total_usd":0.00665,"input_uncached_usd":0.0029000000000000002,"input_cached_usd":0.0,"output_usd":0.0037500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1257678339025006}
{"item_id":"senseval2.d001.s027.t004","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":2,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":3,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"},{"index":4,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"treatment%1:04:00::","call_ids":["senseval2.d001.s027.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.004545,"input_uncached_usd":0.0026550000000000002,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9546742920065299}
{"item_id":"senseval2.d001.s027.t005","gold_sense_keys":["basic%3:00:00::","basic%5:00:02:first:00"],"candidates":[{"index":1,"sense_key":"basic%5:00:00:standard:01","synset_id":"basic.s.01"},{"index":2,"sense_key":"basic%5:00:02:first:00","synset_id":"basic.s.02"},{"index":3,"sense_key":"basic%3:00:00::","synset_id":"basic.a.01"},{"index":4,"sense_key":"basic%5:00:00:alkaline:00","synset_id":"basic.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"basic%3:00:00::","call_ids":["senseval2.d001.s027.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"basic%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":120},"cost":{"currency":"USD","total_usd":0.006700000000000001,"input_uncached_usd":0.0026200000000000004,"input_cached_usd":0.0,"output_usd":0.00408,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.309541042079218}
{"item_id":"senseval2.d001.s027.t007","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":2,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":3,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"},{"index":4,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":5,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":6,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":7,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":8,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":9,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":10,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"begin%2:30:00::","call_ids":["senseval2.d001.s027.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"begin%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":863,"cached_input_tokens":0,"output_tokens":312,"reasoning_output_tokens":296},"cost":{"currency":"USD","total_usd":0.013675,"input_uncached_usd":0.004315,"input_cached_usd":0.0,"output_usd":0.00936,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.921558624948375}
{"item_id":"senseval2.d001.s027.t008","gold_sense_keys":["show%2:39:00::"],"candidates":[{"index":1,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":2,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":3,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":4,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":5,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":6,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":7,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":8,"sense_key":"show%2:33:00::","synset_id":"show.v.12"},{"index":9,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":10,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":11,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":12,"sense_key":"show%2:32:02::","synset_id":"express.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"show%2:39:00::","call_ids":["senseval2.d001.s027.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"show%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":926,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1025},"cost":{"currency":"USD","total_usd":0.03601,"input_uncached_usd":0.0046300000000000004,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":18.543333292007446}
{"item_id":"senseval2.d001.s027.t009","gold_sense_keys":["result%1:11:00::","result%1:19:00::"],"candidates":[{"index":1,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":2,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":3,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"},{"index":4,"sense_key":"result%1:11:00::","synset_id":"result.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"result%1:19:00::","call_ids":["senseval2.d001.s027.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"result%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":1017,"reasoning_output_tokens":995},"cost":{"currency":"USD","total_usd":0.033335000000000004,"input_uncached_usd":0.0028250000000000003,"input_cached_usd":0.0,"output_usd":0.030510000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":17.316353583009914}
{"item_id":"senseval2.d001.s027.t010","gold_sense_keys":["there%4:02:02::"],"candidates":[{"index":1,"sense_key":"there%4:02:01::","synset_id":"there.r.03"},{"index":2,"sense_key":"there%4:02:00::","synset_id":"there.r.01"},{"index":3,"sense_key":"there%4:02:02::","synset_id":"there.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"there%4:02:02::","call_ids":["senseval2.d001.s027.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"there%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":185,"reasoning_output_tokens":169},"cost":{"currency":"USD","total_usd":0.00792,"input_uncached_usd":0.00237,"input_cached_usd":0.0,"output_usd":0.00555,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8008620409527794}
{"item_id":"senseval2.d001.s029.t000","gold_sense_keys":["make%2:36:12::"],"candidates":[{"index":1,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":2,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":3,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":4,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":5,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":6,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":7,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":8,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":9,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":10,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":11,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":12,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":13,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":14,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":15,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":16,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":17,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":18,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":19,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":20,"sense_key":"make%2:29:01::","synset_id":"make.v.49"},{"index":21,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":22,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":23,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":24,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":25,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":26,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":27,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":28,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":29,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":30,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":31,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":32,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":33,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":36,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":37,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":38,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":39,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":40,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":43,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":44,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":45,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":46,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":47,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":48,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":49,"sense_key":"make%2:38:10::","synset_id":"make.v.36"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"make%2:36:12::","call_ids":["senseval2.d001.s029.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"make%2:36:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2292,"cached_input_tokens":0,"output_tokens":425,"reasoning_output_tokens":409},"cost":{"currency":"USD","total_usd":0.024210000000000002,"input_uncached_usd":0.011460000000000001,"input_cached_usd":0.0,"output_usd":0.012750000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.764712042058818}
{"item_id":"senseval2.d001.s029.t002","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":3,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discovery%1:04:00::","call_ids":["senseval2.d001.s029.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discovery%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018165,"input_uncached_usd":0.0023250000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.789278082898818}
{"item_id":"senseval2.d001.s030.t000","gold_sense_keys":["then%4:02:00::"],"candidates":[{"index":1,"sense_key":"then%4:02:01::","synset_id":"then.r.03"},{"index":2,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":3,"sense_key":"then%4:02:00::","synset_id":"then.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"then%4:02:00::","call_ids":["senseval2.d001.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"then%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":160,"reasoning_output_tokens":144},"cost":{"currency":"USD","total_usd":0.007175000000000001,"input_uncached_usd":0.0023750000000000004,"input_cached_usd":0.0,"output_usd":0.0048000000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.553597250021994}
{"item_id":"senseval2.d001.s030.t001","gold_sense_keys":["last%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"last%5:00:00:unalterable:00","synset_id":"final.s.03"},{"index":2,"sense_key":"last%5:00:00:unlikely:00","synset_id":"last.s.03"},{"index":3,"sense_key":"last%5:00:00:past:00","synset_id":"last.s.01"},{"index":4,"sense_key":"last%5:00:00:closing:00","synset_id":"concluding.s.01"},{"index":5,"sense_key":"last%5:00:00:worst:00","synset_id":"last.s.08"},{"index":6,"sense_key":"last%5:00:00:dying:00","synset_id":"last.s.04"},{"index":7,"sense_key":"last%5:00:00:high:02","synset_id":"last.s.06"},{"index":8,"sense_key":"last%3:00:00::","synset_id":"last.a.02"},{"index":9,"sense_key":"last%5:00:00:ultimate:00","synset_id":"final.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"last%5:00:00:past:00","call_ids":["senseval2.d001.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"last%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":759,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.005775000000000001,"input_uncached_usd":0.003795,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.819893499952741}
{"item_id":"senseval2.d001.s030.t002","gold_sense_keys":["spring%1:28:00::"],"candidates":[{"index":1,"sense_key":"spring%1:28:00::","synset_id":"spring.n.01"},{"index":2,"sense_key":"spring%1:07:00::","synset_id":"give.n.01"},{"index":3,"sense_key":"spring%1:06:00::","synset_id":"spring.n.02"},{"index":4,"sense_key":"spring%1:17:00::","synset_id":"spring.n.03"},{"index":5,"sense_key":"spring%1:04:00::","synset_id":"leap.n.01"},{"index":6,"sense_key":"spring%1:15:00::","synset_id":"spring.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"spring%1:28:00::","call_ids":["senseval2.d001.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"spring%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.004215,"input_uncached_usd":0.0028650000000000004,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5784572080010548}
{"item_id":"senseval2.d001.s030.t004","gold_sense_keys":["report%2:32:04::"],"candidates":[{"index":1,"sense_key":"report%2:32:01::","synset_id":"report.v.04"},{"index":2,"sense_key":"report%2:32:00::","synset_id":"report.v.01"},{"index":3,"sense_key":"report%2:32:03::","synset_id":"report.v.03"},{"index":4,"sense_key":"report%2:32:04::","synset_id":"report.v.02"},{"index":5,"sense_key":"report%2:32:02::","synset_id":"report.v.06"},{"index":6,"sense_key":"report%2:32:13::","synset_id":"report.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"report%2:32:04::","call_ids":["senseval2.d001.s030.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"report%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004985,"input_uncached_usd":0.0030050000000000003,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5439378750743344}
{"item_id":"senseval2.d001.s030.t005","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"},{"index":2,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":3,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":4,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":5,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":6,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":7,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":8,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":9,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":12,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"find%2:36:00::","call_ids":["senseval2.d001.s030.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"find%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1138,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02153,"input_uncached_usd":0.005690000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.360926166991703}
{"item_id":"senseval2.d001.s030.t007","gold_sense_keys":["call%2:32:02::"],"candidates":[{"index":1,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":2,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":3,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":4,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":5,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":6,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":7,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":8,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":9,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":10,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":11,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":12,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":13,"sense_key":"call%2:29:05::","synset_id":"call.v.28"},{"index":14,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":15,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":16,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":17,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":18,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":19,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":20,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":21,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":22,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":23,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":24,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":25,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":26,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":27,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":28,"sense_key":"call%2:32:15::","synset_id":"call.v.23"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"call%2:32:02::","call_ids":["senseval2.d001.s030.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"call%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1524,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.009630000000000001,"input_uncached_usd":0.007620000000000001,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.456373833003454}
{"item_id":"senseval2.d001.s030.t008","gold_sense_keys":["impair%2:30:00::"],"candidates":[{"index":1,"sense_key":"impair%2:30:00::","synset_id":"impair.v.01"},{"index":2,"sense_key":"impair%2:30:01::","synset_id":"mar.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"impair%2:30:00::","call_ids":["senseval2.d001.s030.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"impair%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.004425,"input_uncached_usd":0.002085,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.037486374960281}
{"item_id":"senseval2.d001.s030.t009","gold_sense_keys":["turn%2:42:13::"],"candidates":[{"index":1,"sense_key":"turn%2:29:00::","synset_id":"twist.v.10"},{"index":2,"sense_key":"turn%2:36:00::","synset_id":"plow.v.01"},{"index":3,"sense_key":"turn%2:38:13::","synset_id":"turn.v.19"},{"index":4,"sense_key":"turn%2:30:03::","synset_id":"turn.v.07"},{"index":5,"sense_key":"turn%2:38:00::","synset_id":"turn.v.01"},{"index":6,"sense_key":"turn%2:33:13::","synset_id":"turn.v.23"},{"index":7,"sense_key":"turn%2:30:04::","synset_id":"change_state.v.01"},{"index":8,"sense_key":"turn%2:35:08::","synset_id":"turn.v.08"},{"index":9,"sense_key":"turn%2:38:02::","synset_id":"turn.v.04"},{"index":10,"sense_key":"turn%2:38:04::","synset_id":"turn.v.09"},{"index":11,"sense_key":"turn%2:35:11::","synset_id":"flex.v.05"},{"index":12,"sense_key":"turn%2:41:10::","synset_id":"turn.v.17"},{"index":13,"sense_key":"turn%2:38:06::","synset_id":"turn.v.11"},{"index":14,"sense_key":"turn%2:35:06::","synset_id":"turn.v.13"},{"index":15,"sense_key":"turn%2:42:00::","synset_id":"become.v.02"},{"index":16,"sense_key":"turn%2:38:01::","synset_id":"turn.v.10"},{"index":17,"sense_key":"turn%2:30:01::","synset_id":"sour.v.01"},{"index":18,"sense_key":"turn%2:38:14::","synset_id":"turn.v.20"},{"index":19,"sense_key":"turn%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":20,"sense_key":"turn%2:42:13::","synset_id":"turn.v.16"},{"index":21,"sense_key":"turn%2:32:14::","synset_id":"call_on.v.01"},{"index":22,"sense_key":"turn%2:30:14::","synset_id":"turn.v.26"},{"index":23,"sense_key":"turn%2:38:03::","synset_id":"turn.v.06"},{"index":24,"sense_key":"turn%2:35:10::","synset_id":"turn.v.22"},{"index":25,"sense_key":"turn%2:30:09::","synset_id":"turn.v.14"},{"index":26,"sense_key":"turn%2:40:13::","synset_id":"turn.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":20,"chosen_sense_key":"turn%2:42:13::","call_ids":["senseval2.d001.s030.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":20,"predicted_sense_key":"turn%2:42:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1471,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":120},"cost":{"currency":"USD","total_usd":0.011435,"input_uncached_usd":0.007355,"input_cached_usd":0.0,"output_usd":0.00408,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.391637208056636}
{"item_id":"senseval2.d001.s030.t010","gold_sense_keys":["healthy%3:00:00::"],"candidates":[{"index":1,"sense_key":"healthy%5:00:00:sound:00","synset_id":"healthy.s.01"},{"index":2,"sense_key":"healthy%5:00:00:reasonable:00","synset_id":"healthy.s.03"},{"index":3,"sense_key":"healthy%5:00:00:considerable:00","synset_id":"goodly.s.01"},{"index":4,"sense_key":"healthy%5:00:00:wholesome:00","synset_id":"healthy.s.02"},{"index":5,"sense_key":"healthy%3:00:00::","synset_id":"healthy.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"healthy%3:00:00::","call_ids":["senseval2.d001.s030.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"healthy%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.005125,"input_uncached_usd":0.002935,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7590466249966994}
{"item_id":"senseval2.d001.s030.t011","gold_sense_keys":["colon%1:08:00::"],"candidates":[{"index":1,"sense_key":"colon%1:15:00::","synset_id":"colon.n.04"},{"index":2,"sense_key":"colon%1:23:01::","synset_id":"colon.n.02"},{"index":3,"sense_key":"colon%1:23:00::","synset_id":"colon.n.03"},{"index":4,"sense_key":"colon%1:08:00::","synset_id":"colon.n.01"},{"index":5,"sense_key":"colon%1:10:00::","synset_id":"colon.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"colon%1:08:00::","call_ids":["senseval2.d001.s030.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"colon%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00461,"input_uncached_usd":0.00272,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8017622909974307}
{"item_id":"senseval2.d001.s030.t012","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":2,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":3,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":4,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":5,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":6,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":7,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s030.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":31,"reasoning_output_tokens":16},"cost":{"currency":"USD","total_usd":0.004025,"input_uncached_usd":0.003095,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2130385839845985}
{"item_id":"senseval2.d001.s030.t013","gold_sense_keys":["cancerous%5:00:00:malignant:02"],"candidates":[{"index":1,"sense_key":"cancerous%5:00:00:malignant:02","synset_id":"cancerous.s.01"},{"index":2,"sense_key":"cancerous%5:00:00:malign:00","synset_id":"cancerous.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancerous%5:00:00:malignant:02","call_ids":["senseval2.d001.s030.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancerous%5:00:00:malignant:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.0037150000000000004,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6653609169879928}
{"item_id":"senseval2.d001.s031.t001","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":2,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":3,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"},{"index":6,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":7,"sense_key":"report%1:10:00::","synset_id":"report.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"report%1:10:03::","call_ids":["senseval2.d001.s031.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":186,"reasoning_output_tokens":170},"cost":{"currency":"USD","total_usd":0.00885,"input_uncached_usd":0.0032700000000000003,"input_cached_usd":0.0,"output_usd":0.00558,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.20803616696503}
{"item_id":"senseval2.d001.s031.t002","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":2,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":3,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":4,"sense_key":"other%3:00:00::","synset_id":"other.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s031.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00444,"input_uncached_usd":0.00252,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.158301582909189}
{"item_id":"senseval2.d001.s031.t004","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["senseval2.d001.s031.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":386,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0036100000000000004,"input_uncached_usd":0.00193,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.184622916043736}
{"item_id":"senseval2.d001.s031.t005","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"},{"index":2,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":3,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval2.d001.s031.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":204,"reasoning_output_tokens":188},"cost":{"currency":"USD","total_usd":0.00838,"input_uncached_usd":0.0022600000000000003,"input_cached_usd":0.0,"output_usd":0.0061200000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.061186958104372}
{"item_id":"senseval2.d001.s031.t006","gold_sense_keys":["same%3:00:00::"],"candidates":[{"index":1,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":2,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":3,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"},{"index":4,"sense_key":"same%3:00:02::","synset_id":"same.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"same%3:00:02::","call_ids":["senseval2.d001.s031.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"same%3:00:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":90},"cost":{"currency":"USD","total_usd":0.00561,"input_uncached_usd":0.0024600000000000004,"input_cached_usd":0.0,"output_usd":0.00315,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5086252499604598}
{"item_id":"senseval2.d001.s031.t007","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s031.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.0035900000000000003,"input_uncached_usd":0.0020900000000000003,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7056239580269903}
{"item_id":"senseval2.d001.s031.t009","gold_sense_keys":["present%3:00:02::"],"candidates":[{"index":1,"sense_key":"present%3:00:01::","synset_id":"present.a.01"},{"index":2,"sense_key":"present%3:00:02::","synset_id":"present.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"present%3:00:02::","call_ids":["senseval2.d001.s031.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"present%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.00468,"input_uncached_usd":0.0020700000000000002,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0348322079516947}
{"item_id":"senseval2.d001.s031.t010","gold_sense_keys":["tissue%1:08:00::"],"candidates":[{"index":1,"sense_key":"tissue%1:08:00::","synset_id":"tissue.n.01"},{"index":2,"sense_key":"tissue%1:27:02::","synset_id":"tissue.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tissue%1:08:00::","call_ids":["senseval2.d001.s031.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tissue%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":393,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.0037050000000000004,"input_uncached_usd":0.001965,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8504779579816386}
{"item_id":"senseval2.d001.s032.t004","gold_sense_keys":["most%4:02:00::"],"candidates":[{"index":1,"sense_key":"most%4:02:00::","synset_id":"most.r.01"},{"index":2,"sense_key":"most%4:02:02::","synset_id":"about.r.07"},{"index":3,"sense_key":"most%4:02:01::","synset_id":"most.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"most%4:02:00::","call_ids":["senseval2.d001.s032.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"most%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.004050000000000001,"input_uncached_usd":0.0022800000000000003,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7040107910288498}
{"item_id":"senseval2.d001.s032.t005","gold_sense_keys":["common%3:00:01::","common%5:00:00:familiar:02"],"candidates":[{"index":1,"sense_key":"common%3:00:02::","synset_id":"common.a.01"},{"index":2,"sense_key":"common%5:00:00:informal:02","synset_id":"common.s.03"},{"index":3,"sense_key":"common%5:00:00:familiar:02","synset_id":"common.s.02"},{"index":4,"sense_key":"common%5:00:00:inferior:02","synset_id":"coarse.s.02"},{"index":5,"sense_key":"common%5:00:00:lowborn:00","synset_id":"common.s.04"},{"index":6,"sense_key":"common%3:00:01::","synset_id":"common.a.02"},{"index":7,"sense_key":"common%5:00:00:unrefined:01","synset_id":"coarse.s.01"},{"index":8,"sense_key":"common%5:00:00:ordinary:00","synset_id":"common.s.07"},{"index":9,"sense_key":"common%5:00:00:shared:00","synset_id":"common.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"common%5:00:00:familiar:02","call_ids":["senseval2.d001.s032.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"common%5:00:00:familiar:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":763,"cached_input_tokens":0,"output_tokens":217,"reasoning_output_tokens":201},"cost":{"currency":"USD","total_usd":0.010325,"input_uncached_usd":0.0038150000000000002,"input_cached_usd":0.0,"output_usd":0.00651,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.530549959046766}
{"item_id":"senseval2.d001.s032.t007","gold_sense_keys":["form%1:09:01::"],"candidates":[{"index":1,"sense_key":"form%1:10:02::","synset_id":"form.n.10"},{"index":2,"sense_key":"form%1:10:01::","synset_id":"form.n.08"},{"index":3,"sense_key":"form%1:07:03::","synset_id":"form.n.11"},{"index":4,"sense_key":"form%1:06:01::","synset_id":"form.n.16"},{"index":5,"sense_key":"form%1:03:00::","synset_id":"shape.n.02"},{"index":6,"sense_key":"form%1:14:02::","synset_id":"form.n.09"},{"index":7,"sense_key":"form%1:09:01::","synset_id":"kind.n.01"},{"index":8,"sense_key":"form%1:14:00::","synset_id":"class.n.02"},{"index":9,"sense_key":"form%1:07:01::","synset_id":"shape.n.01"},{"index":10,"sense_key":"form%1:09:00::","synset_id":"form.n.03"},{"index":11,"sense_key":"form%1:08:00::","synset_id":"human_body.n.01"},{"index":12,"sense_key":"form%1:06:00::","synset_id":"mannequin.n.02"},{"index":13,"sense_key":"form%1:07:02::","synset_id":"form.n.14"},{"index":14,"sense_key":"form%1:26:00::","synset_id":"phase.n.02"},{"index":15,"sense_key":"form%1:10:00::","synset_id":"form.n.01"},{"index":16,"sense_key":"form%1:07:00::","synset_id":"form.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"form%1:09:01::","call_ids":["senseval2.d001.s032.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"form%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1055,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":122},"cost":{"currency":"USD","total_usd":0.009415,"input_uncached_usd":0.005275,"input_cached_usd":0.0,"output_usd":0.0041400000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.436908750096336}
{"item_id":"senseval2.d001.s032.t010","gold_sense_keys":["kill%2:35:02::"],"candidates":[{"index":1,"sense_key":"kill%2:35:06::","synset_id":"kill.v.07"},{"index":2,"sense_key":"kill%2:39:00::","synset_id":"kill.v.05"},{"index":3,"sense_key":"kill%2:30:08::","synset_id":"stamp_out.v.01"},{"index":4,"sense_key":"kill%2:30:03::","synset_id":"kill.v.14"},{"index":5,"sense_key":"kill%2:41:01::","synset_id":"kill.v.02"},{"index":6,"sense_key":"kill%2:35:02::","synset_id":"kill.v.09"},{"index":7,"sense_key":"kill%2:30:02::","synset_id":"kill.v.15"},{"index":8,"sense_key":"kill%2:37:00::","synset_id":"kill.v.06"},{"index":9,"sense_key":"kill%2:42:00::","synset_id":"kill.v.04"},{"index":10,"sense_key":"kill%2:35:05::","synset_id":"kill.v.08"},{"index":11,"sense_key":"kill%2:35:01::","synset_id":"kill.v.10"},{"index":12,"sense_key":"kill%2:35:00::","synset_id":"kill.v.01"},{"index":13,"sense_key":"kill%2:30:00::","synset_id":"kill.v.12"},{"index":14,"sense_key":"kill%2:30:04::","synset_id":"kill.v.13"},{"index":15,"sense_key":"kill%2:34:00::","synset_id":"toss_off.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"kill%2:35:02::","call_ids":["senseval2.d001.s032.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"kill%2:35:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":461,"reasoning_output_tokens":445},"cost":{"currency":"USD","total_usd":0.018325,"input_uncached_usd":0.004495000000000001,"input_cached_usd":0.0,"output_usd":0.01383,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.076373375020921}
{"item_id":"senseval2.d001.s032.t012","gold_sense_keys":["american%1:18:00::"],"candidates":[{"index":1,"sense_key":"american%1:10:00::","synset_id":"american_english.n.01"},{"index":2,"sense_key":"american%1:18:01::","synset_id":"american.n.03"},{"index":3,"sense_key":"american%1:18:00::","synset_id":"american.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"american%1:18:00::","call_ids":["senseval2.d001.s032.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"american%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.00477,"input_uncached_usd":0.0020700000000000002,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1448940829141065}
{"item_id":"senseval2.d001.s032.t013","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d001.s032.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.004805,"input_uncached_usd":0.002585,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.241859041969292}
{"item_id":"senseval2.d001.s033.t000","gold_sense_keys":["right%4:02:04::","right%4:02:08::"],"candidates":[{"index":1,"sense_key":"right%4:02:05::","synset_id":"justly.r.02"},{"index":2,"sense_key":"right%4:02:06::","synset_id":"mighty.r.01"},{"index":3,"sense_key":"right%4:02:00::","synset_id":"right.r.06"},{"index":4,"sense_key":"right%4:02:04::","synset_id":"right.r.01"},{"index":5,"sense_key":"right%4:02:07::","synset_id":"right.r.07"},{"index":6,"sense_key":"right%4:02:09::","synset_id":"right.r.03"},{"index":7,"sense_key":"right%4:02:01::","synset_id":"properly.r.01"},{"index":8,"sense_key":"right%4:02:03::","synset_id":"right.r.04"},{"index":9,"sense_key":"right%4:02:02::","synset_id":"correctly.r.01"},{"index":10,"sense_key":"right%4:02:08::","synset_id":"right.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"right%4:02:04::","call_ids":["senseval2.d001.s033.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"right%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":251,"reasoning_output_tokens":236},"cost":{"currency":"USD","total_usd":0.010875000000000001,"input_uncached_usd":0.0033450000000000003,"input_cached_usd":0.0,"output_usd":0.00753,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.9435515000950545}
{"item_id":"senseval2.d001.s033.t001","gold_sense_keys":["now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:06::","synset_id":"now.r.07"},{"index":2,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":3,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":4,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":5,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":6,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":7,"sense_key":"now%4:02:04::","synset_id":"now.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"now%4:02:04::","call_ids":["senseval2.d001.s033.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"now%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":603,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.005775000000000001,"input_uncached_usd":0.0030150000000000003,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.020062332972884}
{"item_id":"senseval2.d001.s033.t002","gold_sense_keys":["about%4:02:00::"],"candidates":[{"index":1,"sense_key":"about%4:02:01::","synset_id":"about.r.03"},{"index":2,"sense_key":"about%4:02:04::","synset_id":"about.r.04"},{"index":3,"sense_key":"about%4:02:03::","synset_id":"about.r.06"},{"index":4,"sense_key":"about%4:02:00::","synset_id":"approximately.r.01"},{"index":5,"sense_key":"about%4:02:05::","synset_id":"about.r.07"},{"index":6,"sense_key":"about%4:02:02::","synset_id":"about.r.05"},{"index":7,"sense_key":"about%4:02:06::","synset_id":"about.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"about%4:02:00::","call_ids":["senseval2.d001.s033.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"about%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.004645,"input_uncached_usd":0.003085,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6955463329795748}
{"item_id":"senseval2.d001.s033.t004","gold_sense_keys":["laboratory%1:06:00::"],"candidates":[{"index":1,"sense_key":"laboratory%1:15:00::","synset_id":"testing_ground.n.01"},{"index":2,"sense_key":"laboratory%1:06:00::","synset_id":"lab.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"laboratory%1:06:00::","call_ids":["senseval2.d001.s033.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"laboratory%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.004145,"input_uncached_usd":0.0020150000000000003,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.363858624943532}
{"item_id":"senseval2.d001.s033.t005","gold_sense_keys":["u.s.%1:15:00::"],"candidates":[{"index":1,"sense_key":"u.s.%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"us%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"us%1:15:00::","call_ids":["senseval2.d001.s033.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"us%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":413,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.0040750000000000005,"input_uncached_usd":0.002065,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0307045419467613}
{"item_id":"senseval2.d001.s033.t008","gold_sense_keys":["race%2:41:03::"],"candidates":[{"index":1,"sense_key":"race%2:38:10::","synset_id":"race.v.04"},{"index":2,"sense_key":"race%2:33:00::","synset_id":"race.v.02"},{"index":3,"sense_key":"race%2:41:03::","synset_id":"race.v.03"},{"index":4,"sense_key":"race%2:38:00::","synset_id":"rush.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"race%2:41:03::","call_ids":["senseval2.d001.s033.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"race%2:41:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.00394,"input_uncached_usd":0.0024100000000000002,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2575193750672042}
{"item_id":"senseval2.d001.s033.t009","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":2,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":3,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":4,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s033.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":468,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.004710000000000001,"input_uncached_usd":0.00234,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1615969580598176}
{"item_id":"senseval2.d001.s034.t000","gold_sense_keys":["about%4:02:00::"],"candidates":[{"index":1,"sense_key":"about%4:02:05::","synset_id":"about.r.07"},{"index":2,"sense_key":"about%4:02:02::","synset_id":"about.r.05"},{"index":3,"sense_key":"about%4:02:06::","synset_id":"about.r.02"},{"index":4,"sense_key":"about%4:02:00::","synset_id":"approximately.r.01"},{"index":5,"sense_key":"about%4:02:03::","synset_id":"about.r.06"},{"index":6,"sense_key":"about%4:02:04::","synset_id":"about.r.04"},{"index":7,"sense_key":"about%4:02:01::","synset_id":"about.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"about%4:02:00::","call_ids":["senseval2.d001.s034.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"about%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.0047550000000000005,"input_uncached_usd":0.0032250000000000004,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6252252080012113}
{"item_id":"senseval2.d001.s035.t001","gold_sense_keys":["say%2:32:01::"],"candidates":[{"index":1,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":2,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":3,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":4,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":5,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":6,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":7,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":8,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":9,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":10,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":11,"sense_key":"say%2:32:08::","synset_id":"say.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d001.s035.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":823,"cached_input_tokens":0,"output_tokens":1829,"reasoning_output_tokens":1794},"cost":{"currency":"USD","total_usd":0.058985,"input_uncached_usd":0.004115000000000001,"input_cached_usd":0.0,"output_usd":0.05487,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":32.1848772080848}
{"item_id":"senseval2.d001.s035.t003","gold_sense_keys":["alone%4:02:01::"],"candidates":[{"index":1,"sense_key":"alone%4:02:01::","synset_id":"alone.r.02"},{"index":2,"sense_key":"alone%4:02:00::","synset_id":"entirely.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"alone%4:02:01::","call_ids":["senseval2.d001.s035.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"alone%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":1045,"reasoning_output_tokens":1023},"cost":{"currency":"USD","total_usd":0.033560000000000006,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.03135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":17.248479124973528}
{"item_id":"senseval2.d001.s035.t004","gold_sense_keys":["combination%1:14:00::"],"candidates":[{"index":1,"sense_key":"combination%1:04:01::","synset_id":"combination.n.06"},{"index":2,"sense_key":"combination%1:14:03::","synset_id":"combination.n.05"},{"index":3,"sense_key":"combination%1:14:00::","synset_id":"combination.n.01"},{"index":4,"sense_key":"combination%1:04:00::","synset_id":"combination.n.07"},{"index":5,"sense_key":"combination%1:14:02::","synset_id":"combination.n.03"},{"index":6,"sense_key":"combination%1:14:04::","synset_id":"combination.n.02"},{"index":7,"sense_key":"combination%1:14:01::","synset_id":"combination.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"combination%1:14:00::","call_ids":["senseval2.d001.s035.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"combination%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":237,"reasoning_output_tokens":221},"cost":{"currency":"USD","total_usd":0.01003,"input_uncached_usd":0.0029200000000000003,"input_cached_usd":0.0,"output_usd":0.00711,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.490497708087787}
{"item_id":"senseval2.d001.s035.t005","gold_sense_keys":["appear%2:39:00::","appear%2:39:01::"],"candidates":[{"index":1,"sense_key":"appear%2:30:02::","synset_id":"appear.v.05"},{"index":2,"sense_key":"appear%2:41:03::","synset_id":"appear.v.07"},{"index":3,"sense_key":"appear%2:39:01::","synset_id":"appear.v.04"},{"index":4,"sense_key":"appear%2:39:00::","synset_id":"look.v.02"},{"index":5,"sense_key":"appear%2:36:00::","synset_id":"appear.v.06"},{"index":6,"sense_key":"appear%2:30:01::","synset_id":"appear.v.03"},{"index":7,"sense_key":"appear%2:30:00::","synset_id":"appear.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"appear%2:39:00::","call_ids":["senseval2.d001.s035.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"appear%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":686,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01927,"input_uncached_usd":0.0034300000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.258609041920863}
{"item_id":"senseval2.d001.s035.t006","gold_sense_keys":["crucial%5:00:00:important:00","crucial%5:00:00:material:02"],"candidates":[{"index":1,"sense_key":"crucial%3:00:00::","synset_id":"crucial.a.01"},{"index":2,"sense_key":"crucial%5:00:00:important:00","synset_id":"all-important.s.01"},{"index":3,"sense_key":"crucial%5:00:00:material:02","synset_id":"crucial.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"crucial%5:00:00:important:00","call_ids":["senseval2.d001.s035.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"crucial%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":482,"reasoning_output_tokens":466},"cost":{"currency":"USD","total_usd":0.01684,"input_uncached_usd":0.00238,"input_cached_usd":0.0,"output_usd":0.01446,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.732862542034127}
{"item_id":"senseval2.d001.s035.t007","gold_sense_keys":["development%1:22:02::"],"candidates":[{"index":1,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":2,"sense_key":"development%1:10:00::","synset_id":"development.n.09"},{"index":3,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":4,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":5,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":6,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":7,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":8,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":9,"sense_key":"development%1:22:00::","synset_id":"development.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"development%1:22:02::","call_ids":["senseval2.d001.s035.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"development%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":817,"cached_input_tokens":0,"output_tokens":199,"reasoning_output_tokens":184},"cost":{"currency":"USD","total_usd":0.010055,"input_uncached_usd":0.004085,"input_cached_usd":0.0,"output_usd":0.0059700000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.280982166994363}
{"item_id":"senseval2.d001.s035.t009","gold_sense_keys":["scourge%1:26:00::"],"candidates":[{"index":1,"sense_key":"scourge%1:18:00::","synset_id":"terror.n.02"},{"index":2,"sense_key":"scourge%1:06:00::","synset_id":"scourge.n.01"},{"index":3,"sense_key":"scourge%1:26:00::","synset_id":"bane.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"scourge%1:26:00::","call_ids":["senseval2.d001.s035.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"scourge%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.003625,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.00132,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8607357080327347}
{"item_id":"senseval2.d001.s035.t010","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":2,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":3,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":4,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s035.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":541,"cached_input_tokens":0,"output_tokens":40,"reasoning_output_tokens":25},"cost":{"currency":"USD","total_usd":0.0039050000000000005,"input_uncached_usd":0.0027050000000000004,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6503118330147117}
{"item_id":"senseval2.d001.s035.t011","gold_sense_keys":["brain%1:08:00::"],"candidates":[{"index":1,"sense_key":"brain%1:09:00::","synset_id":"brain.n.02"},{"index":2,"sense_key":"brain%1:18:00::","synset_id":"genius.n.01"},{"index":3,"sense_key":"brain%1:09:01::","synset_id":"mind.n.01"},{"index":4,"sense_key":"brain%1:08:00::","synset_id":"brain.n.01"},{"index":5,"sense_key":"brain%1:13:00::","synset_id":"brain.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"brain%1:08:00::","call_ids":["senseval2.d001.s035.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"brain%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.00448,"input_uncached_usd":0.00286,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9598443750292063}
{"item_id":"senseval2.d001.s035.t012","gold_sense_keys":["skin%1:08:00::"],"candidates":[{"index":1,"sense_key":"skin%1:06:01::","synset_id":"skin.n.02"},{"index":2,"sense_key":"skin%1:13:00::","synset_id":"peel.n.02"},{"index":3,"sense_key":"skin%1:26:00::","synset_id":"skin.n.04"},{"index":4,"sense_key":"skin%1:08:00::","synset_id":"skin.n.01"},{"index":5,"sense_key":"skin%1:05:01::","synset_id":"hide.n.02"},{"index":6,"sense_key":"skin%1:06:02::","synset_id":"skin.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"skin%1:08:00::","call_ids":["senseval2.d001.s035.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"skin%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.004880000000000001,"input_uncached_usd":0.00272,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6487638329854235}
{"item_id":"senseval2.d001.s035.t015","gold_sense_keys":["cervix%1:08:02::"],"candidates":[{"index":1,"sense_key":"cervix%1:08:02::","synset_id":"cervix.n.02"},{"index":2,"sense_key":"cervix%1:08:01::","synset_id":"neck.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cervix%1:08:02::","call_ids":["senseval2.d001.s035.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cervix%1:08:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":436,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.00419,"input_uncached_usd":0.00218,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.134860916994512}
{"item_id":"senseval2.d001.s036.t000","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":2,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"},{"index":3,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval2.d001.s036.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":155,"reasoning_output_tokens":139},"cost":{"currency":"USD","total_usd":0.00706,"input_uncached_usd":0.0024100000000000002,"input_cached_usd":0.0,"output_usd":0.0046500000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.240706375101581}
{"item_id":"senseval2.d001.s036.t001","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval2.d001.s036.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":155,"reasoning_output_tokens":139},"cost":{"currency":"USD","total_usd":0.007340000000000001,"input_uncached_usd":0.00269,"input_cached_usd":0.0,"output_usd":0.0046500000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.747348750010133}
{"item_id":"senseval2.d001.s036.t002","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"},{"index":2,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"},{"index":3,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s036.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.0038650000000000004,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7809375840006396}
{"item_id":"senseval2.d001.s036.t004","gold_sense_keys":["version%1:09:01::"],"candidates":[{"index":1,"sense_key":"version%1:04:00::","synset_id":"version.n.06"},{"index":2,"sense_key":"version%1:10:01::","synset_id":"version.n.01"},{"index":3,"sense_key":"version%1:10:02::","synset_id":"adaptation.n.01"},{"index":4,"sense_key":"version%1:10:00::","synset_id":"translation.n.01"},{"index":5,"sense_key":"version%1:09:00::","synset_id":"interpretation.n.01"},{"index":6,"sense_key":"version%1:09:01::","synset_id":"version.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"version%1:09:01::","call_ids":["senseval2.d001.s036.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"version%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.004935,"input_uncached_usd":0.0030450000000000004,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.289335208013654}
{"item_id":"senseval2.d001.s036.t006","gold_sense_keys":["especially%4:02:00::"],"candidates":[{"index":1,"sense_key":"especially%4:02:00::","synset_id":"particularly.r.01"},{"index":2,"sense_key":"especially%4:02:01::","synset_id":"specially.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"especially%4:02:00::","call_ids":["senseval2.d001.s036.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"especially%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.003985000000000001,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.08654204197228}
{"item_id":"senseval2.d001.s036.t007","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":3,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s036.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":39,"reasoning_output_tokens":24},"cost":{"currency":"USD","total_usd":0.0039700000000000004,"input_uncached_usd":0.0028000000000000004,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4934182919096202}
{"item_id":"senseval2.d001.s036.t009","gold_sense_keys":["explain%2:32:02::"],"candidates":[{"index":1,"sense_key":"explain%2:32:00::","synset_id":"explain.v.01"},{"index":2,"sense_key":"explain%2:32:02::","synset_id":"excuse.v.03"},{"index":3,"sense_key":"explain%2:32:01::","synset_id":"explain.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"explain%2:32:02::","call_ids":["senseval2.d001.s036.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"explain%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.00541,"input_uncached_usd":0.0024700000000000004,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.149199999985285}
{"item_id":"senseval2.d001.s036.t010","gold_sense_keys":["finally%4:02:02::"],"candidates":[{"index":1,"sense_key":"finally%4:02:03::","synset_id":"last.r.02"},{"index":2,"sense_key":"finally%4:02:02::","synset_id":"finally.r.01"},{"index":3,"sense_key":"finally%4:02:00::","synset_id":"ultimately.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"finally%4:02:00::","call_ids":["senseval2.d001.s036.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"finally%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":5481,"reasoning_output_tokens":5404},"cost":{"currency":"USD","total_usd":0.16689,"input_uncached_usd":0.0024600000000000004,"input_cached_usd":0.0,"output_usd":0.16443,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":113.45511587499641}
{"item_id":"senseval2.d001.s036.t011","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":3,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s036.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":561,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.004875000000000001,"input_uncached_usd":0.002805,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0057335420278832}
{"item_id":"senseval2.d001.s036.t012","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":2,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"},{"index":3,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":4,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval2.d001.s036.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"seem%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":1446,"reasoning_output_tokens":1418},"cost":{"currency":"USD","total_usd":0.0461,"input_uncached_usd":0.00272,"input_cached_usd":0.0,"output_usd":0.04338,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":25.838178167003207}
{"item_id":"senseval2.d001.s036.t014","gold_sense_keys":["certain%5:00:00:definite:00"],"candidates":[{"index":1,"sense_key":"certain%5:00:00:careful:00","synset_id":"certain.s.03"},{"index":2,"sense_key":"certain%3:00:05::","synset_id":"sealed.a.01"},{"index":3,"sense_key":"certain%3:00:02::","synset_id":"certain.a.02"},{"index":4,"sense_key":"certain%5:00:00:definite:00","synset_id":"certain.s.01"},{"index":5,"sense_key":"certain%3:00:03::","synset_id":"certain.a.04"},{"index":6,"sense_key":"certain%5:00:00:reliable:00","synset_id":"certain.s.02"},{"index":7,"sense_key":"certain%3:00:01::","synset_id":"certain.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"certain%5:00:00:definite:00","call_ids":["senseval2.d001.s036.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"certain%5:00:00:definite:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":682,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.005810000000000001,"input_uncached_usd":0.0034100000000000003,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4673006660304964}
{"item_id":"senseval2.d001.s036.t015","gold_sense_keys":["family%1:14:01::"],"candidates":[{"index":1,"sense_key":"family%1:18:00::","synset_id":"kin.n.01"},{"index":2,"sense_key":"family%1:14:02::","synset_id":"family.n.01"},{"index":3,"sense_key":"family%1:14:05::","synset_id":"family.n.08"},{"index":4,"sense_key":"family%1:14:04::","synset_id":"class.n.01"},{"index":5,"sense_key":"family%1:14:03::","synset_id":"family.n.06"},{"index":6,"sense_key":"family%1:14:06::","synset_id":"syndicate.n.01"},{"index":7,"sense_key":"family%1:14:01::","synset_id":"family.n.04"},{"index":8,"sense_key":"family%1:14:00::","synset_id":"family.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"family%1:14:01::","call_ids":["senseval2.d001.s036.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"family%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.00673,"input_uncached_usd":0.0036400000000000004,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7137755000730976}
{"item_id":"senseval2.d001.s037.t000","gold_sense_keys":["story%1:10:04::"],"candidates":[{"index":1,"sense_key":"story%1:06:00::","synset_id":"floor.n.02"},{"index":2,"sense_key":"story%1:10:02::","synset_id":"fib.n.01"},{"index":3,"sense_key":"story%1:10:01::","synset_id":"report.n.03"},{"index":4,"sense_key":"story%1:10:03::","synset_id":"narrative.n.01"},{"index":5,"sense_key":"story%1:10:04::","synset_id":"history.n.02"},{"index":6,"sense_key":"story%1:10:00::","synset_id":"story.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"story%1:10:04::","call_ids":["senseval2.d001.s037.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"story%1:10:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":682,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00536,"input_uncached_usd":0.0034100000000000003,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.095531333005056}
{"item_id":"senseval2.d001.s037.t002","gold_sense_keys":["go_back%2:42:00::"],"candidates":[{"index":1,"sense_key":"go_back%2:30:00::","synset_id":"recover.v.03"},{"index":2,"sense_key":"go_back%2:42:00::","synset_id":"go_back.v.01"},{"index":3,"sense_key":"go_back%2:32:00::","synset_id":"recur.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"go_back%2:42:00::","call_ids":["senseval2.d001.s037.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"go_back%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.004425,"input_uncached_usd":0.002445,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8434252500301227}
{"item_id":"senseval2.d001.s037.t005","gold_sense_keys":["name%2:32:03::"],"candidates":[{"index":1,"sense_key":"name%2:32:04::","synset_id":"mention.v.01"},{"index":2,"sense_key":"name%2:31:01::","synset_id":"identify.v.05"},{"index":3,"sense_key":"name%2:31:00::","synset_id":"diagnose.v.01"},{"index":4,"sense_key":"name%2:32:07::","synset_id":"name.v.05"},{"index":5,"sense_key":"name%2:32:03::","synset_id":"name.v.01"},{"index":6,"sense_key":"name%2:41:00::","synset_id":"appoint.v.01"},{"index":7,"sense_key":"name%2:32:09::","synset_id":"list.v.01"},{"index":8,"sense_key":"name%2:32:02::","synset_id":"name.v.02"},{"index":9,"sense_key":"name%2:41:01::","synset_id":"name.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"name%2:32:03::","call_ids":["senseval2.d001.s037.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"name%2:32:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":750,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01959,"input_uncached_usd":0.0037500000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.42054729105439}
{"item_id":"senseval2.d001.s037.t006","gold_sense_keys":["propose%2:31:00::"],"candidates":[{"index":1,"sense_key":"propose%2:31:01::","synset_id":"aim.v.02"},{"index":2,"sense_key":"propose%2:41:00::","synset_id":"nominate.v.02"},{"index":3,"sense_key":"propose%2:32:00::","synset_id":"propose.v.01"},{"index":4,"sense_key":"propose%2:31:00::","synset_id":"project.v.08"},{"index":5,"sense_key":"propose%2:32:01::","synset_id":"propose.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"propose%2:31:00::","call_ids":["senseval2.d001.s037.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"propose%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":94},"cost":{"currency":"USD","total_usd":0.00636,"input_uncached_usd":0.0030600000000000002,"input_cached_usd":0.0,"output_usd":0.0033,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3432963750092313}
{"item_id":"senseval2.d001.s037.t008","gold_sense_keys":["stem%2:42:00::"],"candidates":[{"index":1,"sense_key":"stem%2:42:00::","synset_id":"stem.v.01"},{"index":2,"sense_key":"stem%2:42:03::","synset_id":"stem.v.02"},{"index":3,"sense_key":"stem%2:30:01::","synset_id":"stem.v.04"},{"index":4,"sense_key":"stem%2:30:00::","synset_id":"stem.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stem%2:42:00::","call_ids":["senseval2.d001.s037.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stem%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.004235000000000001,"input_uncached_usd":0.0025550000000000004,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7115276250988245}
{"item_id":"senseval2.d001.s037.t009","gold_sense_keys":["separate%3:00:00::"],"candidates":[{"index":1,"sense_key":"separate%5:00:00:detached:00","synset_id":"freestanding.s.01"},{"index":2,"sense_key":"separate%3:00:00::","synset_id":"separate.a.01"},{"index":3,"sense_key":"separate%5:00:02:segregated:00","synset_id":"separate.s.02"},{"index":4,"sense_key":"separate%5:00:02:unconnected:00","synset_id":"disjoined.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"separate%3:00:00::","call_ids":["senseval2.d001.s037.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"separate%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004485,"input_uncached_usd":0.0026850000000000003,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8454351659165695}
{"item_id":"senseval2.d001.s037.t010","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":2,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":3,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"},{"index":4,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s037.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":540,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":138},"cost":{"currency":"USD","total_usd":0.0072900000000000005,"input_uncached_usd":0.0027,"input_cached_usd":0.0,"output_usd":0.00459,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0804274580441415}
{"item_id":"senseval2.d001.s037.t011","gold_sense_keys":["defect%1:26:01::"],"candidates":[{"index":1,"sense_key":"defect%1:26:00::","synset_id":"defect.n.03"},{"index":2,"sense_key":"defect%1:07:00::","synset_id":"blemish.n.01"},{"index":3,"sense_key":"defect%1:26:01::","synset_id":"defect.n.01"},{"index":4,"sense_key":"defect%1:07:01::","synset_id":"defect.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"defect%1:26:01::","call_ids":["senseval2.d001.s037.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"defect%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":225,"reasoning_output_tokens":209},"cost":{"currency":"USD","total_usd":0.009475,"input_uncached_usd":0.0027250000000000004,"input_cached_usd":0.0,"output_usd":0.00675,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8384377079783008}
{"item_id":"senseval2.d001.s038.t000","gold_sense_keys":["theorize%2:31:00::"],"candidates":[{"index":1,"sense_key":"theorize%2:31:02::","synset_id":"theorize.v.02"},{"index":2,"sense_key":"theorize%2:31:01::","synset_id":"theorize.v.03"},{"index":3,"sense_key":"theorize%2:31:00::","synset_id":"speculate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"theorize%2:31:00::","call_ids":["senseval2.d001.s038.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"theorize%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":312,"reasoning_output_tokens":297},"cost":{"currency":"USD","total_usd":0.011765000000000001,"input_uncached_usd":0.002405,"input_cached_usd":0.0,"output_usd":0.00936,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.840911666047759}
{"item_id":"senseval2.d001.s038.t001","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval2.d001.s038.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.004654999999999999,"input_uncached_usd":0.002735,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0281123750610277}
{"item_id":"senseval2.d001.s038.t002","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":2,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":3,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":4,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":5,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s038.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":550,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.004490000000000001,"input_uncached_usd":0.0027500000000000003,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.126834207912907}
{"item_id":"senseval2.d001.s038.t004","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"},{"index":2,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"},{"index":3,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s038.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.0037300000000000002,"input_uncached_usd":0.0022600000000000003,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6539562080288306}
{"item_id":"senseval2.d001.s038.t005","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"},{"index":2,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s038.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":438,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.00387,"input_uncached_usd":0.00219,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6422098749317229}
{"item_id":"senseval2.d001.s038.t006","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"},{"index":3,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":4,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s038.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.00517,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.4895446250448}
{"item_id":"senseval2.d001.s038.t008","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d001.s038.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":80},"cost":{"currency":"USD","total_usd":0.004985,"input_uncached_usd":0.002135,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.019926500041038}
{"item_id":"senseval2.d001.s038.t009","gold_sense_keys":["normal%3:00:01::"],"candidates":[{"index":1,"sense_key":"normal%3:00:02::","synset_id":"normal.a.02"},{"index":2,"sense_key":"normal%3:00:01::","synset_id":"normal.a.01"},{"index":3,"sense_key":"normal%5:00:00:perpendicular:00","synset_id":"normal.s.01"},{"index":4,"sense_key":"normal%3:00:03::","synset_id":"normal.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"normal%3:00:01::","call_ids":["senseval2.d001.s038.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"normal%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.004835000000000001,"input_uncached_usd":0.002435,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3705572920152918}
{"item_id":"senseval2.d001.s038.t010","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":2,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"},{"index":3,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":4,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s038.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":122},"cost":{"currency":"USD","total_usd":0.00664,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.0041400000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.759161208057776}
{"item_id":"senseval2.d001.s039.t002","gold_sense_keys":["develop%2:30:00::","develop%2:42:00::"],"candidates":[{"index":1,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"},{"index":2,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":3,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":4,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":5,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":6,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":7,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":8,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":9,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":10,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":11,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":12,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":13,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":14,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":15,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":16,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":17,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":18,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":21,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"develop%2:42:00::","call_ids":["senseval2.d001.s039.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"develop%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1283,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.022255,"input_uncached_usd":0.0064150000000000006,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.525181625038385}
{"item_id":"senseval2.d001.s039.t003","gold_sense_keys":["second%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"second%3:00:00::","synset_id":"second.a.02"},{"index":2,"sense_key":"second%5:00:00:ordinal:00","synset_id":"second.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"second%5:00:00:ordinal:00","call_ids":["senseval2.d001.s039.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"second%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.0042699999999999995,"input_uncached_usd":0.00214,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7873222910566255}
{"item_id":"senseval2.d001.s039.t004","gold_sense_keys":["normal%3:00:01::"],"candidates":[{"index":1,"sense_key":"normal%3:00:03::","synset_id":"normal.a.03"},{"index":2,"sense_key":"normal%3:00:01::","synset_id":"normal.a.01"},{"index":3,"sense_key":"normal%5:00:00:perpendicular:00","synset_id":"normal.s.01"},{"index":4,"sense_key":"normal%3:00:02::","synset_id":"normal.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"normal%3:00:01::","call_ids":["senseval2.d001.s039.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"normal%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.005600000000000001,"input_uncached_usd":0.00239,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3989970000693575}
{"item_id":"senseval2.d001.s039.t005","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":4,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s039.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.004945,"input_uncached_usd":0.002455,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9770697909407318}
{"item_id":"senseval2.d001.s039.t007","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s039.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00394,"input_uncached_usd":0.00214,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9940101660322398}
{"item_id":"senseval2.d001.s040.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":2,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":3,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":4,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":5,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":6,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":9,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":10,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":11,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:04:01::","call_ids":["senseval2.d001.s040.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":842,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":148},"cost":{"currency":"USD","total_usd":0.0091,"input_uncached_usd":0.00421,"input_cached_usd":0.0,"output_usd":0.00489,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.437016750103794}
{"item_id":"senseval2.d001.s040.t001","gold_sense_keys":["prove%2:31:00::"],"candidates":[{"index":1,"sense_key":"prove%2:31:02::","synset_id":"prove.v.04"},{"index":2,"sense_key":"prove%2:42:00::","synset_id":"prove.v.01"},{"index":3,"sense_key":"prove%2:31:00::","synset_id":"prove.v.02"},{"index":4,"sense_key":"prove%2:38:01::","synset_id":"rise.v.15"},{"index":5,"sense_key":"prove%2:41:03::","synset_id":"test.v.01"},{"index":6,"sense_key":"prove%2:36:00::","synset_id":"prove.v.08"},{"index":7,"sense_key":"prove%2:32:00::","synset_id":"testify.v.02"},{"index":8,"sense_key":"prove%2:31:01::","synset_id":"prove.v.09"},{"index":9,"sense_key":"prove%2:38:00::","synset_id":"raise.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"prove%2:31:00::","call_ids":["senseval2.d001.s040.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"prove%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":748,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":120},"cost":{"currency":"USD","total_usd":0.00782,"input_uncached_usd":0.0037400000000000003,"input_cached_usd":0.0,"output_usd":0.00408,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3776431248988956}
{"item_id":"senseval2.d001.s040.t002","gold_sense_keys":["theory%1:09:01::"],"candidates":[{"index":1,"sense_key":"theory%1:09:01::","synset_id":"hypothesis.n.02"},{"index":2,"sense_key":"theory%1:09:02::","synset_id":"theory.n.03"},{"index":3,"sense_key":"theory%1:09:00::","synset_id":"theory.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"theory%1:09:01::","call_ids":["senseval2.d001.s040.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"theory%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":88},"cost":{"currency":"USD","total_usd":0.005915,"input_uncached_usd":0.0028250000000000003,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.841698416043073}
{"item_id":"senseval2.d001.s041.t000","gold_sense_keys":["back%4:02:01::"],"candidates":[{"index":1,"sense_key":"back%4:02:01::","synset_id":"back.r.04"},{"index":2,"sense_key":"back%4:02:05::","synset_id":"back.r.03"},{"index":3,"sense_key":"back%4:02:03::","synset_id":"back.r.06"},{"index":4,"sense_key":"back%4:02:04::","synset_id":"back.r.01"},{"index":5,"sense_key":"back%4:02:00::","synset_id":"back.r.02"},{"index":6,"sense_key":"back%4:02:06::","synset_id":"back.r.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"back%4:02:01::","call_ids":["senseval2.d001.s041.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"back%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":562,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.00443,"input_uncached_usd":0.0028100000000000004,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9899830420035869}
{"item_id":"senseval2.d001.s041.t001","gold_sense_keys":["then%4:02:01::"],"candidates":[{"index":1,"sense_key":"then%4:02:01::","synset_id":"then.r.03"},{"index":2,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":3,"sense_key":"then%4:02:02::","synset_id":"then.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"then%4:02:01::","call_ids":["senseval2.d001.s041.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"then%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.0037,"input_uncached_usd":0.00232,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8782013749005273}
{"item_id":"senseval2.d001.s041.t003","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":2,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":3,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":4,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":5,"sense_key":"way%1:21:00::","synset_id":"way.n.12"},{"index":6,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":7,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":8,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":9,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":10,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":11,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":12,"sense_key":"way%1:07:02::","synset_id":"way.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"way%1:04:01::","call_ids":["senseval2.d001.s041.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":807,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.006855,"input_uncached_usd":0.004035,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1792451669462025}
{"item_id":"senseval2.d001.s041.t008","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":2,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":3,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":4,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":5,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":6,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":7,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":8,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":9,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":10,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"},{"index":11,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":12,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":13,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":14,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":15,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":16,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":19,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":20,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":21,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":22,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":23,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":24,"sense_key":"see%2:41:12::","synset_id":"see.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval2.d001.s041.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1480,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":110},"cost":{"currency":"USD","total_usd":0.01118,"input_uncached_usd":0.0074,"input_cached_usd":0.0,"output_usd":0.00378,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0805242089554667}
{"item_id":"senseval2.d001.s041.t009","gold_sense_keys":["pair%1:14:01::"],"candidates":[{"index":1,"sense_key":"pair%1:14:00::","synset_id":"pair.n.03"},{"index":2,"sense_key":"pair%1:14:01::","synset_id":"pair.n.01"},{"index":3,"sense_key":"pair%1:14:02::","synset_id":"pair.n.04"},{"index":4,"sense_key":"pair%1:23:00::","synset_id":"couple.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pair%1:14:01::","call_ids":["senseval2.d001.s041.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pair%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":255,"reasoning_output_tokens":239},"cost":{"currency":"USD","total_usd":0.009930000000000001,"input_uncached_usd":0.0022800000000000003,"input_cached_usd":0.0,"output_usd":0.0076500000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.907498707994819}
{"item_id":"senseval2.d001.s041.t011","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":3,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":4,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":5,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":6,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":7,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s041.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.0043950000000000005,"input_uncached_usd":0.0030450000000000004,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8539393750252202}
{"item_id":"senseval2.d001.s041.t012","gold_sense_keys":["contain%2:42:13::"],"candidates":[{"index":1,"sense_key":"contain%2:42:00::","synset_id":"incorporate.v.02"},{"index":2,"sense_key":"contain%2:41:00::","synset_id":"control.v.02"},{"index":3,"sense_key":"contain%2:33:00::","synset_id":"check.v.18"},{"index":4,"sense_key":"contain%2:42:06::","synset_id":"contain.v.04"},{"index":5,"sense_key":"contain%2:42:14::","synset_id":"contain.v.05"},{"index":6,"sense_key":"contain%2:42:13::","synset_id":"hold.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contain%2:42:00::","call_ids":["senseval2.d001.s041.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contain%2:42:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.03457,"input_uncached_usd":0.00319,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":20.440463583916426}
{"item_id":"senseval2.d001.s042.t001","gold_sense_keys":["gross%5:00:00:visible:00"],"candidates":[{"index":1,"sense_key":"gross%3:00:00::","synset_id":"gross.a.01"},{"index":2,"sense_key":"gross%5:00:00:conspicuous:00","synset_id":"crying.s.02"},{"index":3,"sense_key":"gross%5:00:00:fat:01","synset_id":"gross.s.02"},{"index":4,"sense_key":"gross%5:00:00:general:00","synset_id":"gross.s.01"},{"index":5,"sense_key":"gross%5:00:00:indecent:00","synset_id":"crude.s.02"},{"index":6,"sense_key":"gross%5:00:00:unmitigated:00","synset_id":"arrant.s.01"},{"index":7,"sense_key":"gross%5:00:00:visible:00","synset_id":"megascopic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"gross%5:00:00:visible:00","call_ids":["senseval2.d001.s042.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"gross%5:00:00:visible:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019085,"input_uncached_usd":0.003245,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.695205040974542}
{"item_id":"senseval2.d001.s042.t003","gold_sense_keys":["damage%1:11:00::"],"candidates":[{"index":1,"sense_key":"damage%1:04:00::","synset_id":"damage.n.03"},{"index":2,"sense_key":"damage%1:11:00::","synset_id":"damage.n.01"},{"index":3,"sense_key":"damage%1:11:01::","synset_id":"damage.n.02"},{"index":4,"sense_key":"damage%1:21:00::","synset_id":"price.n.02"},{"index":5,"sense_key":"damage%1:04:01::","synset_id":"wrong.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"damage%1:11:00::","call_ids":["senseval2.d001.s042.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"damage%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.0053,"input_uncached_usd":0.00254,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2425775410374627}
{"item_id":"senseval2.d001.s042.t004","gold_sense_keys":["visible%3:00:00::"],"candidates":[{"index":1,"sense_key":"visible%5:00:00:available:00","synset_id":"visible.s.02"},{"index":2,"sense_key":"visible%5:00:00:overt:00","synset_id":"visible.s.01"},{"index":3,"sense_key":"visible%3:00:00::","synset_id":"visible.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"visible%3:00:00::","call_ids":["senseval2.d001.s042.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"visible%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":269,"reasoning_output_tokens":253},"cost":{"currency":"USD","total_usd":0.010335,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.00807,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.4452100419439375}
{"item_id":"senseval2.d001.s043.t000","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":2,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":3,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"},{"index":4,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":5,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":6,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":7,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":8,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":9,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":10,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":11,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":12,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":13,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":14,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":15,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":16,"sense_key":"find%2:38:10::","synset_id":"find.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"find%2:31:09::","call_ids":["senseval2.d001.s043.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"find%2:31:09::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1100,"cached_input_tokens":0,"output_tokens":1105,"reasoning_output_tokens":1078},"cost":{"currency":"USD","total_usd":0.03865,"input_uncached_usd":0.0055000000000000005,"input_cached_usd":0.0,"output_usd":0.03315,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":21.687591042020358}
{"item_id":"senseval2.d001.s043.t001","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:07::","synset_id":"child.n.04"},{"index":2,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":3,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":4,"sense_key":"child%1:18:00::","synset_id":"child.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"child%1:18:00::","call_ids":["senseval2.d001.s043.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"child%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":310,"reasoning_output_tokens":294},"cost":{"currency":"USD","total_usd":0.01171,"input_uncached_usd":0.0024100000000000002,"input_cached_usd":0.0,"output_usd":0.009300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.306723791989498}
{"item_id":"senseval2.d001.s043.t002","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":5,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval2.d001.s043.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.0045650000000000005,"input_uncached_usd":0.0025250000000000003,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.986246749991551}
{"item_id":"senseval2.d001.s043.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":2,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":3,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":4,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":5,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s043.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.0038950000000000005,"input_uncached_usd":0.0025450000000000004,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7825840000296012}
{"item_id":"senseval2.d001.s043.t004","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"},{"index":2,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"},{"index":3,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s043.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.0036100000000000004,"input_uncached_usd":0.00205,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2929462909232825}
{"item_id":"senseval2.d001.s043.t005","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s043.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.00414,"input_uncached_usd":0.00198,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3710073339752853}
{"item_id":"senseval2.d001.s043.t006","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":3,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"},{"index":4,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s043.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":458,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.004660000000000001,"input_uncached_usd":0.0022900000000000004,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.489280291949399}
{"item_id":"senseval2.d001.s043.t009","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"},{"index":2,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d001.s043.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":385,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.003755,"input_uncached_usd":0.001925,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9344267909182236}
{"item_id":"senseval2.d001.s044.t001","gold_sense_keys":["actually%4:02:01::"],"candidates":[{"index":1,"sense_key":"actually%4:02:00::","synset_id":"actually.r.02"},{"index":2,"sense_key":"actually%4:02:03::","synset_id":"actually.r.03"},{"index":3,"sense_key":"actually%4:02:02::","synset_id":"actually.r.04"},{"index":4,"sense_key":"actually%4:02:01::","synset_id":"actually.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"actually%4:02:01::","call_ids":["senseval2.d001.s044.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"actually%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":406,"reasoning_output_tokens":390},"cost":{"currency":"USD","total_usd":0.01459,"input_uncached_usd":0.0024100000000000002,"input_cached_usd":0.0,"output_usd":0.01218,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.720849332981743}
{"item_id":"senseval2.d001.s044.t002","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":2,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":3,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"},{"index":4,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":5,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":6,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":7,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":8,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":9,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":10,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":11,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":12,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":13,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":14,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":15,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":16,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":17,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":18,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":19,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":20,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":21,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":22,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":23,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":24,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval2.d001.s044.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1445,"cached_input_tokens":0,"output_tokens":303,"reasoning_output_tokens":287},"cost":{"currency":"USD","total_usd":0.016315,"input_uncached_usd":0.0072250000000000005,"input_cached_usd":0.0,"output_usd":0.00909,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.095557915978134}
{"item_id":"senseval2.d001.s044.t003","gold_sense_keys":["bit%1:17:00::","bit%1:23:01::"],"candidates":[{"index":1,"sense_key":"bit%1:13:00::","synset_id":"morsel.n.02"},{"index":2,"sense_key":"bit%1:06:02::","synset_id":"bit.n.10"},{"index":3,"sense_key":"bit%1:10:00::","synset_id":"act.n.04"},{"index":4,"sense_key":"bit%1:10:01::","synset_id":"snatch.n.01"},{"index":5,"sense_key":"bit%1:28:00::","synset_id":"moment.n.02"},{"index":6,"sense_key":"bit%1:23:00::","synset_id":"bit.n.06"},{"index":7,"sense_key":"bit%1:06:01::","synset_id":"bit.n.05"},{"index":8,"sense_key":"bit%1:17:00::","synset_id":"bit.n.02"},{"index":9,"sense_key":"bit%1:06:00::","synset_id":"bit.n.11"},{"index":10,"sense_key":"bit%1:11:00::","synset_id":"piece.n.05"},{"index":11,"sense_key":"bit%1:23:01::","synset_id":"spot.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"bit%1:17:00::","call_ids":["senseval2.d001.s044.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"bit%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1023},"cost":{"currency":"USD","total_usd":0.03532500000000001,"input_uncached_usd":0.003945000000000001,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":17.512977292062715}
{"item_id":"senseval2.d001.s045.t000","gold_sense_keys":["assume%2:31:00::"],"candidates":[{"index":1,"sense_key":"assume%2:40:01::","synset_id":"bear.v.06"},{"index":2,"sense_key":"assume%2:29:00::","synset_id":"wear.v.09"},{"index":3,"sense_key":"assume%2:40:00::","synset_id":"assume.v.06"},{"index":4,"sense_key":"assume%2:35:00::","synset_id":"assume.v.08"},{"index":5,"sense_key":"assume%2:38:00::","synset_id":"assume.v.05"},{"index":6,"sense_key":"assume%2:30:00::","synset_id":"assume.v.03"},{"index":7,"sense_key":"assume%2:36:00::","synset_id":"simulate.v.03"},{"index":8,"sense_key":"assume%2:31:00::","synset_id":"assume.v.01"},{"index":9,"sense_key":"assume%2:41:00::","synset_id":"assume.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"assume%2:31:00::","call_ids":["senseval2.d001.s045.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"assume%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.00574,"input_uncached_usd":0.00385,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.586191541980952}
{"item_id":"senseval2.d001.s045.t002","gold_sense_keys":["piece%1:06:00::","piece%1:17:00::"],"candidates":[{"index":1,"sense_key":"piece%1:06:04::","synset_id":"objet_d'art.n.01"},{"index":2,"sense_key":"piece%1:06:03::","synset_id":"firearm.n.01"},{"index":3,"sense_key":"piece%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"piece%1:07:00::","synset_id":"piece.n.09"},{"index":5,"sense_key":"piece%1:28:00::","synset_id":"while.n.01"},{"index":6,"sense_key":"piece%1:06:00::","synset_id":"piece.n.01"},{"index":7,"sense_key":"piece%1:10:02::","synset_id":"piece.n.06"},{"index":8,"sense_key":"piece%1:13:00::","synset_id":"piece.n.08"},{"index":9,"sense_key":"piece%1:06:02::","synset_id":"man.n.10"},{"index":10,"sense_key":"piece%1:21:00::","synset_id":"slice.n.01"},{"index":11,"sense_key":"piece%1:11:00::","synset_id":"piece.n.05"},{"index":12,"sense_key":"piece%1:10:01::","synset_id":"musical_composition.n.01"},{"index":13,"sense_key":"piece%1:06:05::","synset_id":"piece.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"piece%1:17:00::","call_ids":["senseval2.d001.s045.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"piece%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":856,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":110},"cost":{"currency":"USD","total_usd":0.00806,"input_uncached_usd":0.00428,"input_cached_usd":0.0,"output_usd":0.00378,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.782074666931294}
{"item_id":"senseval2.d001.s045.t003","gold_sense_keys":["contain%2:42:00::"],"candidates":[{"index":1,"sense_key":"contain%2:42:13::","synset_id":"hold.v.11"},{"index":2,"sense_key":"contain%2:42:14::","synset_id":"contain.v.05"},{"index":3,"sense_key":"contain%2:42:00::","synset_id":"incorporate.v.02"},{"index":4,"sense_key":"contain%2:41:00::","synset_id":"control.v.02"},{"index":5,"sense_key":"contain%2:33:00::","synset_id":"check.v.18"},{"index":6,"sense_key":"contain%2:42:06::","synset_id":"contain.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"contain%2:42:00::","call_ids":["senseval2.d001.s045.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"contain%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":409,"reasoning_output_tokens":393},"cost":{"currency":"USD","total_usd":0.01526,"input_uncached_usd":0.00299,"input_cached_usd":0.0,"output_usd":0.01227,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.153380500036292}
{"item_id":"senseval2.d001.s045.t006","gold_sense_keys":["loss%1:07:00::","loss%1:21:01::"],"candidates":[{"index":1,"sense_key":"loss%1:11:01::","synset_id":"loss.n.05"},{"index":2,"sense_key":"loss%1:11:02::","synset_id":"passing.n.02"},{"index":3,"sense_key":"loss%1:11:00::","synset_id":"personnel_casualty.n.01"},{"index":4,"sense_key":"loss%1:04:00::","synset_id":"loss.n.03"},{"index":5,"sense_key":"loss%1:21:00::","synset_id":"loss.n.06"},{"index":6,"sense_key":"loss%1:21:01::","synset_id":"loss.n.01"},{"index":7,"sense_key":"loss%1:07:00::","synset_id":"loss.n.04"},{"index":8,"sense_key":"loss%1:22:00::","synset_id":"loss.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"loss%1:04:00::","call_ids":["senseval2.d001.s045.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"loss%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":3935,"reasoning_output_tokens":3876},"cost":{"currency":"USD","total_usd":0.12117,"input_uncached_usd":0.0031200000000000004,"input_cached_usd":0.0,"output_usd":0.11805,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":74.37477374996524}
{"item_id":"senseval2.d001.s045.t007","gold_sense_keys":["critical%5:00:00:indispensable:00"],"candidates":[{"index":1,"sense_key":"critical%5:00:00:crucial:00","synset_id":"critical.s.02"},{"index":2,"sense_key":"critical%3:00:02::","synset_id":"critical.a.03"},{"index":3,"sense_key":"critical%3:00:01::","synset_id":"critical.a.01"},{"index":4,"sense_key":"critical%3:00:04::","synset_id":"critical.a.02"},{"index":5,"sense_key":"critical%3:00:03::","synset_id":"critical.a.06"},{"index":6,"sense_key":"critical%5:00:00:indispensable:00","synset_id":"critical.s.01"},{"index":7,"sense_key":"critical%3:01:00::","synset_id":"critical.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"critical%5:00:00:indispensable:00","call_ids":["senseval2.d001.s045.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"critical%5:00:00:indispensable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01881,"input_uncached_usd":0.0029700000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.157045374973677}
{"item_id":"senseval2.d001.s045.t008","gold_sense_keys":["role%1:07:00::"],"candidates":[{"index":1,"sense_key":"role%1:04:01::","synset_id":"role.n.04"},{"index":2,"sense_key":"role%1:04:00::","synset_id":"function.n.03"},{"index":3,"sense_key":"role%1:09:00::","synset_id":"character.n.04"},{"index":4,"sense_key":"role%1:07:00::","synset_id":"function.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"role%1:07:00::","call_ids":["senseval2.d001.s045.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"role%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":191,"reasoning_output_tokens":176},"cost":{"currency":"USD","total_usd":0.00817,"input_uncached_usd":0.0024400000000000003,"input_cached_usd":0.0,"output_usd":0.00573,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.717082459013909}
{"item_id":"senseval2.d001.s045.t009","gold_sense_keys":["set_off%2:36:00::"],"candidates":[{"index":1,"sense_key":"set_off%2:38:00::","synset_id":"depart.v.03"},{"index":2,"sense_key":"set_off%2:30:02::","synset_id":"bring_out.v.04"},{"index":3,"sense_key":"set_off%2:42:00::","synset_id":"cancel.v.02"},{"index":4,"sense_key":"set_off%2:30:00::","synset_id":"explode.v.01"},{"index":5,"sense_key":"set_off%2:30:01::","synset_id":"set_off.v.06"},{"index":6,"sense_key":"set_off%2:36:00::","synset_id":"trip.v.04"},{"index":7,"sense_key":"set_off%2:32:00::","synset_id":"incite.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"set_off%2:36:00::","call_ids":["senseval2.d001.s045.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"set_off%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":524,"reasoning_output_tokens":508},"cost":{"currency":"USD","total_usd":0.018770000000000002,"input_uncached_usd":0.00305,"input_cached_usd":0.0,"output_usd":0.01572,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.799132707994431}
{"item_id":"senseval2.d001.s045.t010","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":2,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":3,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":4,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":5,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s045.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.0040999999999999995,"input_uncached_usd":0.00248,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5475009579677135}
{"item_id":"senseval2.d001.s046.t000","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":2,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":3,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":4,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":5,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":6,"sense_key":"know%2:31:14::","synset_id":"know.v.11"},{"index":7,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":8,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":9,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":10,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":11,"sense_key":"know%2:31:12::","synset_id":"know.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d001.s046.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":816,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":122},"cost":{"currency":"USD","total_usd":0.008220000000000002,"input_uncached_usd":0.00408,"input_cached_usd":0.0,"output_usd":0.0041400000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4952063329983503}
{"item_id":"senseval2.d001.s047.t000","gold_sense_keys":["then%4:02:00::"],"candidates":[{"index":1,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":2,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":3,"sense_key":"then%4:02:01::","synset_id":"then.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"then%4:02:00::","call_ids":["senseval2.d001.s047.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"then%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":425,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":81},"cost":{"currency":"USD","total_usd":0.005005,"input_uncached_usd":0.002125,"input_cached_usd":0.0,"output_usd":0.00288,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.068310417002067}
{"item_id":"senseval2.d001.s047.t001","gold_sense_keys":["scientific%3:01:00::"],"candidates":[{"index":1,"sense_key":"scientific%3:00:00::","synset_id":"scientific.a.02"},{"index":2,"sense_key":"scientific%3:01:00::","synset_id":"scientific.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"scientific%3:01:00::","call_ids":["senseval2.d001.s047.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"scientific%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":363,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.0050550000000000005,"input_uncached_usd":0.0018150000000000002,"input_cached_usd":0.0,"output_usd":0.0032400000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.404314166982658}
{"item_id":"senseval2.d001.s047.t002","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["senseval2.d001.s047.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":353,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.0031450000000000002,"input_uncached_usd":0.001765,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7151360829593614}
{"item_id":"senseval2.d001.s047.t003","gold_sense_keys":["lead%2:41:00::"],"candidates":[{"index":1,"sense_key":"lead%2:38:02::","synset_id":"precede.v.04"},{"index":2,"sense_key":"lead%2:41:00::","synset_id":"head.v.02"},{"index":3,"sense_key":"lead%2:36:00::","synset_id":"conduct.v.02"},{"index":4,"sense_key":"lead%2:42:00::","synset_id":"run.v.03"},{"index":5,"sense_key":"lead%2:42:04::","synset_id":"leave.v.07"},{"index":6,"sense_key":"lead%2:32:01::","synset_id":"moderate.v.01"},{"index":7,"sense_key":"lead%2:38:01::","synset_id":"lead.v.01"},{"index":8,"sense_key":"lead%2:42:12::","synset_id":"lead.v.03"},{"index":9,"sense_key":"lead%2:42:03::","synset_id":"go.v.25"},{"index":10,"sense_key":"lead%2:32:00::","synset_id":"lead.v.05"},{"index":11,"sense_key":"lead%2:38:00::","synset_id":"lead.v.04"},{"index":12,"sense_key":"lead%2:42:01::","synset_id":"lead.v.08"},{"index":13,"sense_key":"lead%2:41:12::","synset_id":"contribute.v.03"},{"index":14,"sense_key":"lead%2:42:02::","synset_id":"run.v.23"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"lead%2:41:00::","call_ids":["senseval2.d001.s047.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"lead%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":878,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.006940000000000001,"input_uncached_usd":0.004390000000000001,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.44822804199066}
{"item_id":"senseval2.d001.s047.t006","gold_sense_keys":["then%4:02:01::"],"candidates":[{"index":1,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":2,"sense_key":"then%4:02:01::","synset_id":"then.r.03"},{"index":3,"sense_key":"then%4:02:02::","synset_id":"then.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"then%4:02:01::","call_ids":["senseval2.d001.s047.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"then%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.004920000000000001,"input_uncached_usd":0.0021300000000000004,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.315445125102997}
{"item_id":"senseval2.d001.s047.t007","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:40:03::"],"candidates":[{"index":1,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"},{"index":2,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":3,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":4,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":5,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":6,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":7,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":8,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":9,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":10,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":11,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":12,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":13,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":14,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:40:02::","synset_id":"find.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"find%2:32:00::","call_ids":["senseval2.d001.s047.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"find%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1089,"cached_input_tokens":0,"output_tokens":261,"reasoning_output_tokens":246},"cost":{"currency":"USD","total_usd":0.013275,"input_uncached_usd":0.005445,"input_cached_usd":0.0,"output_usd":0.00783,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.0936426249099895}
{"item_id":"senseval2.d001.s047.t008","gold_sense_keys":["answer%1:10:02::"],"candidates":[{"index":1,"sense_key":"answer%1:10:02::","synset_id":"solution.n.02"},{"index":2,"sense_key":"answer%1:04:00::","synset_id":"answer.n.05"},{"index":3,"sense_key":"answer%1:10:00::","synset_id":"answer.n.03"},{"index":4,"sense_key":"answer%1:10:01::","synset_id":"answer.n.01"},{"index":5,"sense_key":"answer%1:10:03::","synset_id":"answer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"answer%1:10:02::","call_ids":["senseval2.d001.s047.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"answer%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.0046700000000000005,"input_uncached_usd":0.00287,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4039622501004487}
{"item_id":"senseval2.d001.s048.t000","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["senseval2.d001.s048.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":375,"cached_input_tokens":0,"output_tokens":36,"reasoning_output_tokens":21},"cost":{"currency":"USD","total_usd":0.002955,"input_uncached_usd":0.0018750000000000001,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.449577166000381}
{"item_id":"senseval2.d001.s048.t001","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":2,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":3,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":4,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":5,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":6,"sense_key":"use%2:34:01::","synset_id":"use.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d001.s048.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.005135000000000001,"input_uncached_usd":0.0030050000000000003,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6334832910215482}
{"item_id":"senseval2.d001.s048.t002","gold_sense_keys":["battery%1:14:01::"],"candidates":[{"index":1,"sense_key":"battery%1:14:02::","synset_id":"battery.n.01"},{"index":2,"sense_key":"battery%1:06:00::","synset_id":"battery.n.02"},{"index":3,"sense_key":"battery%1:06:03::","synset_id":"battery.n.05"},{"index":4,"sense_key":"battery%1:14:01::","synset_id":"battery.n.03"},{"index":5,"sense_key":"battery%1:04:00::","synset_id":"battery.n.07"},{"index":6,"sense_key":"battery%1:14:03::","synset_id":"battery.n.04"},{"index":7,"sense_key":"battery%1:04:01::","synset_id":"barrage.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"battery%1:14:01::","call_ids":["senseval2.d001.s048.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"battery%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.004595,"input_uncached_usd":0.002915,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8657225840725005}
{"item_id":"senseval2.d001.s048.t004","gold_sense_keys":["develop%2:36:01::"],"candidates":[{"index":1,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":2,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":3,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":4,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":5,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":6,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":7,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":8,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":11,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":12,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":13,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":14,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":15,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":16,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"},{"index":17,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":18,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":19,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":20,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":21,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"develop%2:36:01::","call_ids":["senseval2.d001.s048.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"develop%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1261,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.008615000000000001,"input_uncached_usd":0.006305000000000001,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.386710499995388}
{"item_id":"senseval2.d001.s048.t007","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":2,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"},{"index":3,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":4,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s048.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":105},"cost":{"currency":"USD","total_usd":0.00609,"input_uncached_usd":0.00249,"input_cached_usd":0.0,"output_usd":0.0036,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.381452125031501}
{"item_id":"senseval2.d001.s048.t008","gold_sense_keys":["material%1:27:00::"],"candidates":[{"index":1,"sense_key":"material%1:06:00::","synset_id":"fabric.n.01"},{"index":2,"sense_key":"material%1:27:00::","synset_id":"material.n.01"},{"index":3,"sense_key":"material%1:10:00::","synset_id":"material.n.02"},{"index":4,"sense_key":"material%1:18:00::","synset_id":"material.n.05"},{"index":5,"sense_key":"material%1:06:01::","synset_id":"material.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"material%1:27:00::","call_ids":["senseval2.d001.s048.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"material%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":166,"reasoning_output_tokens":150},"cost":{"currency":"USD","total_usd":0.007775000000000001,"input_uncached_usd":0.002795,"input_cached_usd":0.0,"output_usd":0.00498,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9816387919709086}
{"item_id":"senseval2.d001.s048.t011","gold_sense_keys":["presence%1:26:00::"],"candidates":[{"index":1,"sense_key":"presence%1:26:00::","synset_id":"presence.n.01"},{"index":2,"sense_key":"presence%1:07:00::","synset_id":"bearing.n.03"},{"index":3,"sense_key":"presence%1:18:00::","synset_id":"presence.n.03"},{"index":4,"sense_key":"presence%1:09:00::","synset_id":"presence.n.04"},{"index":5,"sense_key":"presence%1:15:00::","synset_id":"presence.n.02"},{"index":6,"sense_key":"presence%1:04:00::","synset_id":"presence.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"presence%1:26:00::","call_ids":["senseval2.d001.s048.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"presence%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.004175,"input_uncached_usd":0.0025550000000000004,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5543357080314308}
{"item_id":"senseval2.d001.s048.t012","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":2,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":3,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":4,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":5,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":6,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":7,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s048.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.004455000000000001,"input_uncached_usd":0.002955,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8959902920760214}
{"item_id":"senseval2.d001.s049.t000","gold_sense_keys":["analyze%2:31:00::","analyze%2:31:01::"],"candidates":[{"index":1,"sense_key":"analyze%2:31:01::","synset_id":"analyze.v.02"},{"index":2,"sense_key":"analyze%2:31:04::","synset_id":"analyze.v.03"},{"index":3,"sense_key":"analyze%2:31:03::","synset_id":"analyze.v.04"},{"index":4,"sense_key":"analyze%2:31:00::","synset_id":"analyze.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"analyze%2:31:01::","call_ids":["senseval2.d001.s049.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"analyze%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":293,"reasoning_output_tokens":277},"cost":{"currency":"USD","total_usd":0.011295,"input_uncached_usd":0.0025050000000000003,"input_cached_usd":0.0,"output_usd":0.008790000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.661425999947824}
{"item_id":"senseval2.d001.s049.t001","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":2,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":3,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":4,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":5,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":6,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":7,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s049.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0047150000000000004,"input_uncached_usd":0.0028250000000000003,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6455322919646278}
{"item_id":"senseval2.d001.s049.t002","gold_sense_keys":["extract%2:35:04::"],"candidates":[{"index":1,"sense_key":"extract%2:35:04::","synset_id":"extract.v.01"},{"index":2,"sense_key":"extract%2:31:00::","synset_id":"excerpt.v.01"},{"index":3,"sense_key":"extract%2:40:00::","synset_id":"extract.v.02"},{"index":4,"sense_key":"extract%2:35:07::","synset_id":"extract.v.05"},{"index":5,"sense_key":"extract%2:35:05::","synset_id":"press_out.v.03"},{"index":6,"sense_key":"extract%2:30:00::","synset_id":"distill.v.03"},{"index":7,"sense_key":"extract%2:31:02::","synset_id":"extract.v.08"},{"index":8,"sense_key":"extract%2:36:00::","synset_id":"educe.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"extract%2:35:04::","call_ids":["senseval2.d001.s049.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"extract%2:35:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.005635,"input_uncached_usd":0.003085,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.115360458032228}
{"item_id":"senseval2.d001.s049.t003","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":2,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":3,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":4,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"},{"index":5,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval2.d001.s049.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004485,"input_uncached_usd":0.002445,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0098238330101594}
{"item_id":"senseval2.d001.s049.t005","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":2,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":3,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":4,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":5,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":10,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":11,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":12,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":13,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"},{"index":14,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":15,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":16,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":16,"chosen_sense_key":"find%2:39:02::","call_ids":["senseval2.d001.s049.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":16,"predicted_sense_key":"find%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1084,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":148},"cost":{"currency":"USD","total_usd":0.01031,"input_uncached_usd":0.00542,"input_cached_usd":0.0,"output_usd":0.00489,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6005927090300247}
{"item_id":"senseval2.d001.s049.t006","gold_sense_keys":["defect%1:26:01::"],"candidates":[{"index":1,"sense_key":"defect%1:07:01::","synset_id":"defect.n.02"},{"index":2,"sense_key":"defect%1:26:01::","synset_id":"defect.n.01"},{"index":3,"sense_key":"defect%1:07:00::","synset_id":"blemish.n.01"},{"index":4,"sense_key":"defect%1:26:00::","synset_id":"defect.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"defect%1:26:01::","call_ids":["senseval2.d001.s049.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"defect%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":137},"cost":{"currency":"USD","total_usd":0.006975,"input_uncached_usd":0.0023850000000000004,"input_cached_usd":0.0,"output_usd":0.00459,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.241071166004986}
{"item_id":"senseval2.d001.s049.t007","gold_sense_keys":["second%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"second%5:00:00:ordinal:00","synset_id":"second.s.01"},{"index":2,"sense_key":"second%3:00:00::","synset_id":"second.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%5:00:00:ordinal:00","call_ids":["senseval2.d001.s049.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":380,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.0036100000000000004,"input_uncached_usd":0.0019000000000000002,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.776617124909535}
{"item_id":"senseval2.d001.s049.t008","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s049.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.00446,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8956502090441063}
{"item_id":"senseval2.d001.s049.t010","gold_sense_keys":["exact%3:00:00::"],"candidates":[{"index":1,"sense_key":"exact%5:00:00:correct:00","synset_id":"accurate.s.01"},{"index":2,"sense_key":"exact%3:00:00::","synset_id":"exact.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"exact%3:00:00::","call_ids":["senseval2.d001.s049.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"exact%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":385,"cached_input_tokens":0,"output_tokens":635,"reasoning_output_tokens":613},"cost":{"currency":"USD","total_usd":0.020975,"input_uncached_usd":0.001925,"input_cached_usd":0.0,"output_usd":0.01905,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.478209124994464}
{"item_id":"senseval2.d001.s049.t012","gold_sense_keys":["first%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"first%5:00:01:opening:00","synset_id":"beginning.s.01"},{"index":2,"sense_key":"first%5:00:02:opening:00","synset_id":"inaugural.s.01"},{"index":3,"sense_key":"first%3:00:00::","synset_id":"first.a.01"},{"index":4,"sense_key":"first%5:00:00:best:00","synset_id":"first.s.04"},{"index":5,"sense_key":"first%5:00:00:ordinal:00","synset_id":"first.s.01"},{"index":6,"sense_key":"first%3:00:03::","synset_id":"first.a.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"first%5:00:00:ordinal:00","call_ids":["senseval2.d001.s049.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"first%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":146},"cost":{"currency":"USD","total_usd":0.007615,"input_uncached_usd":0.002755,"input_cached_usd":0.0,"output_usd":0.00486,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5975635000504553}
{"item_id":"senseval2.d001.s049.t013","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":3,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s049.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.00419,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.439671791042201}
{"item_id":"senseval2.d001.s050.t000","gold_sense_keys":["finding%1:17:00::"],"candidates":[{"index":1,"sense_key":"finding%1:04:01::","synset_id":"finding.n.02"},{"index":2,"sense_key":"finding%1:17:00::","synset_id":"finding.n.03"},{"index":3,"sense_key":"finding%1:04:00::","synset_id":"determination.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"finding%1:17:00::","call_ids":["senseval2.d001.s050.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"finding%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":113},"cost":{"currency":"USD","total_usd":0.005965,"input_uncached_usd":0.002095,"input_cached_usd":0.0,"output_usd":0.00387,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9666575000155717}
{"item_id":"senseval2.d001.s050.t001","gold_sense_keys":["medicine%1:04:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":3,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"},{"index":4,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:04:00::","call_ids":["senseval2.d001.s050.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":97},"cost":{"currency":"USD","total_usd":0.00576,"input_uncached_usd":0.00237,"input_cached_usd":0.0,"output_usd":0.0033900000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.202122916933149}
{"item_id":"senseval2.d001.s051.t000","gold_sense_keys":["first%3:00:00::"],"candidates":[{"index":1,"sense_key":"first%3:00:00::","synset_id":"first.a.01"},{"index":2,"sense_key":"first%3:00:03::","synset_id":"first.a.06"},{"index":3,"sense_key":"first%5:00:01:opening:00","synset_id":"beginning.s.01"},{"index":4,"sense_key":"first%5:00:00:best:00","synset_id":"first.s.04"},{"index":5,"sense_key":"first%5:00:02:opening:00","synset_id":"inaugural.s.01"},{"index":6,"sense_key":"first%5:00:00:ordinal:00","synset_id":"first.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"first%3:00:00::","call_ids":["senseval2.d001.s051.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"first%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":564,"cached_input_tokens":0,"output_tokens":460,"reasoning_output_tokens":444},"cost":{"currency":"USD","total_usd":0.01662,"input_uncached_usd":0.00282,"input_cached_usd":0.0,"output_usd":0.0138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.47388374991715}
{"item_id":"senseval2.d001.s051.t001","gold_sense_keys":["time%1:11:00::"],"candidates":[{"index":1,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":2,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":3,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":4,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":5,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":6,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":7,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":8,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"},{"index":9,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":10,"sense_key":"time%1:03:00::","synset_id":"time.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"time%1:11:00::","call_ids":["senseval2.d001.s051.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"time%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.005860000000000001,"input_uncached_usd":0.00355,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.715121667017229}
{"item_id":"senseval2.d001.s051.t002","gold_sense_keys":["show%2:31:00::","show%2:32:00::"],"candidates":[{"index":1,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":2,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":3,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":4,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":5,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":6,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":7,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":8,"sense_key":"show%2:33:00::","synset_id":"show.v.12"},{"index":9,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":10,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":11,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":12,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"show%2:31:00::","call_ids":["senseval2.d001.s051.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"show%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":876,"cached_input_tokens":0,"output_tokens":219,"reasoning_output_tokens":203},"cost":{"currency":"USD","total_usd":0.010950000000000001,"input_uncached_usd":0.00438,"input_cached_usd":0.0,"output_usd":0.00657,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.650866582989693}
{"item_id":"senseval2.d001.s051.t003","gold_sense_keys":["loss%1:04:00::"],"candidates":[{"index":1,"sense_key":"loss%1:04:00::","synset_id":"loss.n.03"},{"index":2,"sense_key":"loss%1:22:00::","synset_id":"loss.n.02"},{"index":3,"sense_key":"loss%1:11:01::","synset_id":"loss.n.05"},{"index":4,"sense_key":"loss%1:11:00::","synset_id":"personnel_casualty.n.01"},{"index":5,"sense_key":"loss%1:21:01::","synset_id":"loss.n.01"},{"index":6,"sense_key":"loss%1:11:02::","synset_id":"passing.n.02"},{"index":7,"sense_key":"loss%1:21:00::","synset_id":"loss.n.06"},{"index":8,"sense_key":"loss%1:07:00::","synset_id":"loss.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"loss%1:04:00::","call_ids":["senseval2.d001.s051.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"loss%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018975000000000002,"input_uncached_usd":0.0031650000000000003,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.634676415938884}
{"item_id":"senseval2.d001.s051.t004","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s051.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.0048850000000000005,"input_uncached_usd":0.002275,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7446809579851106}
{"item_id":"senseval2.d001.s051.t005","gold_sense_keys":["same%3:00:02::"],"candidates":[{"index":1,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":2,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":3,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"same%3:00:02::","call_ids":["senseval2.d001.s051.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"same%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":389,"reasoning_output_tokens":373},"cost":{"currency":"USD","total_usd":0.014005,"input_uncached_usd":0.0023350000000000003,"input_cached_usd":0.0,"output_usd":0.01167,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.235736207920127}
{"item_id":"senseval2.d001.s051.t007","gold_sense_keys":["lead%2:41:12::","lead%2:42:12::"],"candidates":[{"index":1,"sense_key":"lead%2:38:00::","synset_id":"lead.v.04"},{"index":2,"sense_key":"lead%2:38:02::","synset_id":"precede.v.04"},{"index":3,"sense_key":"lead%2:32:01::","synset_id":"moderate.v.01"},{"index":4,"sense_key":"lead%2:42:02::","synset_id":"run.v.23"},{"index":5,"sense_key":"lead%2:42:01::","synset_id":"lead.v.08"},{"index":6,"sense_key":"lead%2:41:00::","synset_id":"head.v.02"},{"index":7,"sense_key":"lead%2:32:00::","synset_id":"lead.v.05"},{"index":8,"sense_key":"lead%2:42:00::","synset_id":"run.v.03"},{"index":9,"sense_key":"lead%2:36:00::","synset_id":"conduct.v.02"},{"index":10,"sense_key":"lead%2:42:04::","synset_id":"leave.v.07"},{"index":11,"sense_key":"lead%2:42:03::","synset_id":"go.v.25"},{"index":12,"sense_key":"lead%2:38:01::","synset_id":"lead.v.01"},{"index":13,"sense_key":"lead%2:42:12::","synset_id":"lead.v.03"},{"index":14,"sense_key":"lead%2:41:12::","synset_id":"contribute.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"lead%2:42:12::","call_ids":["senseval2.d001.s051.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"lead%2:42:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":886,"cached_input_tokens":0,"output_tokens":143,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.008720000000000002,"input_uncached_usd":0.004430000000000001,"input_cached_usd":0.0,"output_usd":0.00429,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6694560000905767}
{"item_id":"senseval2.d001.s051.t008","gold_sense_keys":["eruption%1:11:03::","eruption%1:26:02::"],"candidates":[{"index":1,"sense_key":"eruption%1:11:00::","synset_id":"volcanic_eruption.n.01"},{"index":2,"sense_key":"eruption%1:11:02::","synset_id":"eruption.n.06"},{"index":3,"sense_key":"eruption%1:26:01::","synset_id":"eruption.n.03"},{"index":4,"sense_key":"eruption%1:26:02::","synset_id":"eruption.n.02"},{"index":5,"sense_key":"eruption%1:11:03::","synset_id":"outbreak.n.01"},{"index":6,"sense_key":"eruption%1:11:01::","synset_id":"bang.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"eruption%1:11:03::","call_ids":["senseval2.d001.s051.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"eruption%1:11:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":113},"cost":{"currency":"USD","total_usd":0.006475,"input_uncached_usd":0.002605,"input_cached_usd":0.0,"output_usd":0.00387,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2697571250610054}
{"item_id":"senseval2.d001.s051.t009","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":2,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":3,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":4,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s051.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":42,"reasoning_output_tokens":27},"cost":{"currency":"USD","total_usd":0.003785,"input_uncached_usd":0.0025250000000000003,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8408532500034198}
{"item_id":"senseval2.d001.s052.t001","gold_sense_keys":["satisfying%5:00:00:satisfactory:00"],"candidates":[{"index":1,"sense_key":"satisfying%5:00:00:wholesome:00","synset_id":"hearty.s.02"},{"index":2,"sense_key":"satisfying%5:00:00:satisfactory:00","synset_id":"comforting.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"satisfying%5:00:00:satisfactory:00","call_ids":["senseval2.d001.s052.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"satisfying%5:00:00:satisfactory:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":394,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.004370000000000001,"input_uncached_usd":0.00197,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.700090374913998}
{"item_id":"senseval2.d001.s052.t002","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":2,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":3,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":4,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":5,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":6,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":7,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":10,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":11,"sense_key":"say%2:32:07::","synset_id":"say.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d001.s052.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":795,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019815,"input_uncached_usd":0.003975,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.407687540980987}
{"item_id":"senseval2.d001.s052.t003","gold_sense_keys":["now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:06::","synset_id":"now.r.07"},{"index":2,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":3,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":4,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":5,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":6,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":7,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"now%4:02:04::","call_ids":["senseval2.d001.s052.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"now%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.006115000000000001,"input_uncached_usd":0.003115,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1573269579093903}
{"item_id":"senseval2.d001.s053.t000","gold_sense_keys":["true%3:00:00::"],"candidates":[{"index":1,"sense_key":"true%5:00:00:typical:00","synset_id":"true.s.03"},{"index":2,"sense_key":"true%5:00:00:accurate:00","synset_id":"true.s.01"},{"index":3,"sense_key":"true%5:00:00:geographic:00","synset_id":"true.s.07"},{"index":4,"sense_key":"true%5:00:00:legitimate:00","synset_id":"true.s.08"},{"index":5,"sense_key":"true%5:00:02:faithful:00","synset_id":"true.s.02"},{"index":6,"sense_key":"true%5:00:00:real:02","synset_id":"true.s.06"},{"index":7,"sense_key":"true%5:00:00:trustworthy:00","synset_id":"dependable.s.01"},{"index":8,"sense_key":"true%3:00:00::","synset_id":"true.a.01"},{"index":9,"sense_key":"true%5:00:00:harmonious:00","synset_id":"on-key.s.01"},{"index":10,"sense_key":"true%5:00:00:sincere:00","synset_id":"genuine.s.01"},{"index":11,"sense_key":"true%5:00:00:even:01","synset_id":"true.s.10"},{"index":12,"sense_key":"true%3:00:04::","synset_id":"truthful.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"true%3:00:00::","call_ids":["senseval2.d001.s053.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"true%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":815,"cached_input_tokens":0,"output_tokens":214,"reasoning_output_tokens":198},"cost":{"currency":"USD","total_usd":0.010495,"input_uncached_usd":0.0040750000000000005,"input_cached_usd":0.0,"output_usd":0.00642,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.455229458049871}
{"item_id":"senseval2.d001.s053.t002","gold_sense_keys":["true%3:00:00::"],"candidates":[{"index":1,"sense_key":"true%5:00:02:faithful:00","synset_id":"true.s.02"},{"index":2,"sense_key":"true%3:00:04::","synset_id":"truthful.a.01"},{"index":3,"sense_key":"true%5:00:00:trustworthy:00","synset_id":"dependable.s.01"},{"index":4,"sense_key":"true%5:00:00:geographic:00","synset_id":"true.s.07"},{"index":5,"sense_key":"true%5:00:00:harmonious:00","synset_id":"on-key.s.01"},{"index":6,"sense_key":"true%5:00:00:real:02","synset_id":"true.s.06"},{"index":7,"sense_key":"true%5:00:00:even:01","synset_id":"true.s.10"},{"index":8,"sense_key":"true%5:00:00:legitimate:00","synset_id":"true.s.08"},{"index":9,"sense_key":"true%5:00:00:sincere:00","synset_id":"genuine.s.01"},{"index":10,"sense_key":"true%3:00:00::","synset_id":"true.a.01"},{"index":11,"sense_key":"true%5:00:00:accurate:00","synset_id":"true.s.01"},{"index":12,"sense_key":"true%5:00:00:typical:00","synset_id":"true.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"true%3:00:00::","call_ids":["senseval2.d001.s053.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"true%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":815,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":119},"cost":{"currency":"USD","total_usd":0.008095000000000001,"input_uncached_usd":0.0040750000000000005,"input_cached_usd":0.0,"output_usd":0.00402,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7921137909870595}
{"item_id":"senseval2.d001.s053.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":2,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":3,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":4,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s053.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.00398,"input_uncached_usd":0.00248,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6743984170025215}
{"item_id":"senseval2.d001.s054.t000","gold_sense_keys":["audacious%5:00:00:adventurous:00"],"candidates":[{"index":1,"sense_key":"audacious%5:00:00:bold:00","synset_id":"audacious.s.01"},{"index":2,"sense_key":"audacious%5:00:00:unashamed:00","synset_id":"audacious.s.02"},{"index":3,"sense_key":"audacious%5:00:00:adventurous:00","synset_id":"audacious.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"audacious%5:00:00:adventurous:00","call_ids":["senseval2.d001.s054.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"audacious%5:00:00:adventurous:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":103},"cost":{"currency":"USD","total_usd":0.006104999999999999,"input_uncached_usd":0.002565,"input_cached_usd":0.0,"output_usd":0.00354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3947507919510826}
{"item_id":"senseval2.d001.s054.t001","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":2,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":3,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"},{"index":4,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":5,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":6,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"claim%1:10:02::","call_ids":["senseval2.d001.s054.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"claim%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.005220000000000001,"input_uncached_usd":0.0027300000000000002,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0325103329960257}
{"item_id":"senseval2.d001.s055.t000","gold_sense_keys":["young%3:00:00::"],"candidates":[{"index":1,"sense_key":"young%3:00:00::","synset_id":"young.a.01"},{"index":2,"sense_key":"young%5:00:01:young:00","synset_id":"youthful.s.01"},{"index":3,"sense_key":"young%5:00:00:inexperienced:00","synset_id":"unseasoned.s.02"},{"index":4,"sense_key":"young%5:00:00:new:00","synset_id":"young.s.03"},{"index":5,"sense_key":"young%5:00:00:early:02","synset_id":"new.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"young%3:00:00::","call_ids":["senseval2.d001.s055.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"young%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":115},"cost":{"currency":"USD","total_usd":0.006575000000000001,"input_uncached_usd":0.002645,"input_cached_usd":0.0,"output_usd":0.00393,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.151082208030857}
{"item_id":"senseval2.d001.s055.t003","gold_sense_keys":["believe%2:31:04::"],"candidates":[{"index":1,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":2,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"},{"index":3,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"},{"index":4,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"},{"index":5,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"believe%2:31:04::","call_ids":["senseval2.d001.s055.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"believe%2:31:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018475,"input_uncached_usd":0.002635,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.60398470889777}
{"item_id":"senseval2.d001.s055.t004","gold_sense_keys":["right%3:00:04::"],"candidates":[{"index":1,"sense_key":"right%5:00:00:opportune:00","synset_id":"good.s.10"},{"index":2,"sense_key":"right%3:00:02::","synset_id":"correct.a.01"},{"index":3,"sense_key":"right%3:00:03::","synset_id":"right.a.07"},{"index":4,"sense_key":"right%5:00:00:outside:00","synset_id":"right.s.07"},{"index":5,"sense_key":"right%3:00:01::","synset_id":"right.a.04"},{"index":6,"sense_key":"right%5:00:00:perpendicular:00","synset_id":"right.s.06"},{"index":7,"sense_key":"right%3:00:04::","synset_id":"right.a.05"},{"index":8,"sense_key":"right%5:00:00:appropriate:00","synset_id":"proper.s.03"},{"index":9,"sense_key":"right%5:00:00:right-handed:00","synset_id":"right.s.04"},{"index":10,"sense_key":"right%5:00:01:proper:00","synset_id":"correct.s.01"},{"index":11,"sense_key":"right%5:00:00:satisfactory:00","synset_id":"right.s.03"},{"index":12,"sense_key":"right%5:00:00:accurate:00","synset_id":"veracious.s.02"},{"index":13,"sense_key":"right%5:00:02:proper:00","synset_id":"correct.s.02"},{"index":14,"sense_key":"right%3:00:00::","synset_id":"right.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"right%3:00:04::","call_ids":["senseval2.d001.s055.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"right%3:00:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":879,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":90},"cost":{"currency":"USD","total_usd":0.007575,"input_uncached_usd":0.0043950000000000005,"input_cached_usd":0.0,"output_usd":0.00318,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6581592499278486}
{"item_id":"senseval2.d001.s055.t005","gold_sense_keys":["set_out%2:30:00::"],"candidates":[{"index":1,"sense_key":"set_out%2:35:00::","synset_id":"range.v.05"},{"index":2,"sense_key":"set_out%2:38:00::","synset_id":"depart.v.03"},{"index":3,"sense_key":"set_out%2:30:00::","synset_id":"get_down.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"set_out%2:30:00::","call_ids":["senseval2.d001.s055.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"set_out%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.004435,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.785897709080018}
{"item_id":"senseval2.d001.s055.t006","gold_sense_keys":["repeat%2:36:00::"],"candidates":[{"index":1,"sense_key":"repeat%2:36:01::","synset_id":"reprise.v.01"},{"index":2,"sense_key":"repeat%2:41:00::","synset_id":"repeat.v.05"},{"index":3,"sense_key":"repeat%2:30:00::","synset_id":"recur.v.01"},{"index":4,"sense_key":"repeat%2:32:01::","synset_id":"repeat.v.04"},{"index":5,"sense_key":"repeat%2:36:00::","synset_id":"duplicate.v.01"},{"index":6,"sense_key":"repeat%2:32:00::","synset_id":"repeat.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"repeat%2:36:00::","call_ids":["senseval2.d001.s055.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"repeat%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":99},"cost":{"currency":"USD","total_usd":0.00618,"input_uncached_usd":0.0027300000000000002,"input_cached_usd":0.0,"output_usd":0.00345,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7905567090492696}
{"item_id":"senseval2.d001.s055.t007","gold_sense_keys":["experiment%1:04:00::"],"candidates":[{"index":1,"sense_key":"experiment%1:04:01::","synset_id":"experiment.n.03"},{"index":2,"sense_key":"experiment%1:09:00::","synset_id":"experiment.n.02"},{"index":3,"sense_key":"experiment%1:04:00::","synset_id":"experiment.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"experiment%1:04:00::","call_ids":["senseval2.d001.s055.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"experiment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":112},"cost":{"currency":"USD","total_usd":0.005895,"input_uncached_usd":0.002085,"input_cached_usd":0.0,"output_usd":0.00381,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1308170419652015}
{"item_id":"senseval2.d001.s055.t008","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":2,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":3,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":4,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":5,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":6,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":7,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s055.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":580,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.00452,"input_uncached_usd":0.0029000000000000002,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8034663340076804}
{"item_id":"senseval2.d001.s055.t009","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":2,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":3,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":4,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s055.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.004085,"input_uncached_usd":0.002405,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.45902237505652}
{"item_id":"senseval2.d001.s055.t010","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":2,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":3,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":4,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s055.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":507,"cached_input_tokens":0,"output_tokens":40,"reasoning_output_tokens":24},"cost":{"currency":"USD","total_usd":0.0037350000000000005,"input_uncached_usd":0.0025350000000000004,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4650649580871686}
{"item_id":"senseval2.d001.s056.t001","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:01::","synset_id":"team.n.02"},{"index":2,"sense_key":"team%1:14:00::","synset_id":"team.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"team%1:14:00::","call_ids":["senseval2.d001.s056.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":382,"cached_input_tokens":0,"output_tokens":39,"reasoning_output_tokens":24},"cost":{"currency":"USD","total_usd":0.0030800000000000003,"input_uncached_usd":0.0019100000000000002,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4438540419796482}
{"item_id":"senseval2.d001.s056.t002","gold_sense_keys":["report%2:32:04::"],"candidates":[{"index":1,"sense_key":"report%2:32:02::","synset_id":"report.v.06"},{"index":2,"sense_key":"report%2:32:01::","synset_id":"report.v.04"},{"index":3,"sense_key":"report%2:32:04::","synset_id":"report.v.02"},{"index":4,"sense_key":"report%2:32:00::","synset_id":"report.v.01"},{"index":5,"sense_key":"report%2:32:03::","synset_id":"report.v.03"},{"index":6,"sense_key":"report%2:32:13::","synset_id":"report.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"report%2:32:04::","call_ids":["senseval2.d001.s056.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"report%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":581,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.0048850000000000005,"input_uncached_usd":0.002905,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5617611669003963}
{"item_id":"senseval2.d001.s056.t003","gold_sense_keys":["dual%5:00:00:multiple:00"],"candidates":[{"index":1,"sense_key":"dual%5:00:00:plural:00","synset_id":"dual.s.03"},{"index":2,"sense_key":"dual%5:00:02:multiple:00","synset_id":"double.s.01"},{"index":3,"sense_key":"dual%5:00:00:multiple:00","synset_id":"double.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"dual%5:00:00:multiple:00","call_ids":["senseval2.d001.s056.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"dual%5:00:00:multiple:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":105},"cost":{"currency":"USD","total_usd":0.0063,"input_uncached_usd":0.00267,"input_cached_usd":0.0,"output_usd":0.00363,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2837570830015466}
{"item_id":"senseval2.d001.s056.t005","gold_sense_keys":["loss%1:04:00::"],"candidates":[{"index":1,"sense_key":"loss%1:11:01::","synset_id":"loss.n.05"},{"index":2,"sense_key":"loss%1:21:00::","synset_id":"loss.n.06"},{"index":3,"sense_key":"loss%1:04:00::","synset_id":"loss.n.03"},{"index":4,"sense_key":"loss%1:21:01::","synset_id":"loss.n.01"},{"index":5,"sense_key":"loss%1:22:00::","synset_id":"loss.n.02"},{"index":6,"sense_key":"loss%1:11:02::","synset_id":"passing.n.02"},{"index":7,"sense_key":"loss%1:07:00::","synset_id":"loss.n.04"},{"index":8,"sense_key":"loss%1:11:00::","synset_id":"personnel_casualty.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"loss%1:04:00::","call_ids":["senseval2.d001.s056.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"loss%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":655,"cached_input_tokens":0,"output_tokens":257,"reasoning_output_tokens":241},"cost":{"currency":"USD","total_usd":0.010985000000000002,"input_uncached_usd":0.003275,"input_cached_usd":0.0,"output_usd":0.00771,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.840802207938395}
{"item_id":"senseval2.d001.s056.t007","gold_sense_keys":["childhood%1:28:00::"],"candidates":[{"index":1,"sense_key":"childhood%1:26:00::","synset_id":"childhood.n.02"},{"index":2,"sense_key":"childhood%1:28:00::","synset_id":"childhood.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"childhood%1:28:00::","call_ids":["senseval2.d001.s056.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"childhood%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":107},"cost":{"currency":"USD","total_usd":0.005595,"input_uncached_usd":0.001935,"input_cached_usd":0.0,"output_usd":0.00366,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.070463667041622}
{"item_id":"senseval2.d001.s056.t008","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":2,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":5,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s056.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.004315,"input_uncached_usd":0.002635,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4330310830846429}
{"item_id":"senseval2.d001.s056.t010","gold_sense_keys":["call%2:32:02::"],"candidates":[{"index":1,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":2,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":3,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":4,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":5,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":6,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":7,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":8,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":9,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":10,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":11,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":12,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":13,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":14,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":15,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":16,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":17,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":18,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":19,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":20,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":21,"sense_key":"call%2:29:05::","synset_id":"call.v.28"},{"index":22,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":23,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":24,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":25,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":26,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":27,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":28,"sense_key":"call%2:32:03::","synset_id":"call.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":20,"chosen_sense_key":"call%2:32:02::","call_ids":["senseval2.d001.s056.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":20,"predicted_sense_key":"call%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1504,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.010520000000000002,"input_uncached_usd":0.007520000000000001,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8880778750171885}
{"item_id":"senseval2.d001.s057.t001","gold_sense_keys":["turn_to%2:30:00::"],"candidates":[{"index":1,"sense_key":"turn_to%2:30:00::","synset_id":"turn_to.v.02"},{"index":2,"sense_key":"turn_to%2:32:01::","synset_id":"address.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"turn_to%2:30:00::","call_ids":["senseval2.d001.s057.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"turn_to%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.0037,"input_uncached_usd":0.00205,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4420662500197068}
{"item_id":"senseval2.d001.s057.t005","gold_sense_keys":["second%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"second%5:00:00:ordinal:00","synset_id":"second.s.01"},{"index":2,"sense_key":"second%3:00:00::","synset_id":"second.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%5:00:00:ordinal:00","call_ids":["senseval2.d001.s057.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":413,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.003925,"input_uncached_usd":0.002065,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6113378329901025}
{"item_id":"senseval2.d001.s057.t006","gold_sense_keys":["big%3:00:01::","big%5:00:00:conspicuous:00"],"candidates":[{"index":1,"sense_key":"big%5:00:00:proud:00","synset_id":"big.s.07"},{"index":2,"sense_key":"big%5:00:00:heavy:08","synset_id":"big.s.09"},{"index":3,"sense_key":"big%3:00:01::","synset_id":"large.a.01"},{"index":4,"sense_key":"big%5:00:00:pregnant:00","synset_id":"big.s.12"},{"index":5,"sense_key":"big%5:00:00:intemperate:00","synset_id":"big.s.05"},{"index":6,"sense_key":"big%5:00:00:generous:01","synset_id":"big.s.11"},{"index":7,"sense_key":"big%5:00:00:important:00","synset_id":"big.s.01"},{"index":8,"sense_key":"big%5:00:00:intense:00","synset_id":"bad.s.01"},{"index":9,"sense_key":"big%5:00:00:conspicuous:00","synset_id":"big.s.04"},{"index":10,"sense_key":"big%5:00:00:mature:01","synset_id":"adult.s.01"},{"index":11,"sense_key":"big%5:00:00:loud:00","synset_id":"big.s.03"},{"index":12,"sense_key":"big%5:00:02:proud:00","synset_id":"boastful.s.01"},{"index":13,"sense_key":"big%5:00:00:generous:02","synset_id":"big.s.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"big%3:00:01::","call_ids":["senseval2.d001.s057.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"big%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":917,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.007045000000000001,"input_uncached_usd":0.0045850000000000005,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3804328329861164}
{"item_id":"senseval2.d001.s057.t007","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":2,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":3,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":4,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":5,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s057.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.00419,"input_uncached_usd":0.0026300000000000004,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2612753750290722}
{"item_id":"senseval2.d001.s057.t008","gold_sense_keys":["killer%1:26:00::"],"candidates":[{"index":1,"sense_key":"killer%1:18:00::","synset_id":"killer.n.01"},{"index":2,"sense_key":"killer%1:09:00::","synset_id":"killer.n.03"},{"index":3,"sense_key":"killer%1:26:00::","synset_id":"cause_of_death.n.01"},{"index":4,"sense_key":"killer%1:05:00::","synset_id":"killer_whale.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"killer%1:26:00::","call_ids":["senseval2.d001.s057.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"killer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.004165,"input_uncached_usd":0.002425,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.354428124963306}
{"item_id":"senseval2.d001.s057.t009","gold_sense_keys":["u.s.%1:15:00::"],"candidates":[{"index":1,"sense_key":"u.s.%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"us%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"us%1:15:00::","call_ids":["senseval2.d001.s057.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"us%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00402,"input_uncached_usd":0.00222,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.1804244999075308}
{"item_id":"senseval2.d001.s058.t000","gold_sense_keys":["believe%2:31:03::"],"candidates":[{"index":1,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":2,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"},{"index":3,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"},{"index":4,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"},{"index":5,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"believe%2:31:04::","call_ids":["senseval2.d001.s058.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"believe%2:31:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":3393,"reasoning_output_tokens":3341},"cost":{"currency":"USD","total_usd":0.104605,"input_uncached_usd":0.0028150000000000002,"input_cached_usd":0.0,"output_usd":0.10179,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":52.559029040974565}
{"item_id":"senseval2.d001.s058.t004","gold_sense_keys":["arise%2:42:00::"],"candidates":[{"index":1,"sense_key":"arise%2:38:00::","synset_id":"arise.v.03"},{"index":2,"sense_key":"arise%2:42:00::","synset_id":"originate.v.01"},{"index":3,"sense_key":"arise%2:38:04::","synset_id":"rise.v.01"},{"index":4,"sense_key":"arise%2:29:00::","synset_id":"get_up.v.02"},{"index":5,"sense_key":"arise%2:30:00::","synset_id":"arise.v.04"},{"index":6,"sense_key":"arise%2:42:01::","synset_id":"arise.v.02"},{"index":7,"sense_key":"arise%2:41:00::","synset_id":"rebel.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"arise%2:30:00::","call_ids":["senseval2.d001.s058.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"arise%2:30:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":454,"reasoning_output_tokens":438},"cost":{"currency":"USD","total_usd":0.01695,"input_uncached_usd":0.00333,"input_cached_usd":0.0,"output_usd":0.01362,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.268908499972895}
{"item_id":"senseval2.d001.s058.t006","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":2,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":3,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":4,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":5,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s058.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.00446,"input_uncached_usd":0.00272,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.02814004092943}
{"item_id":"senseval2.d001.s058.t008","gold_sense_keys":["often%4:02:00::","often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"often%4:02:02::","call_ids":["senseval2.d001.s058.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"often%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":158,"reasoning_output_tokens":142},"cost":{"currency":"USD","total_usd":0.007010000000000001,"input_uncached_usd":0.0022700000000000003,"input_cached_usd":0.0,"output_usd":0.00474,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8151043748948723}
{"item_id":"senseval2.d001.s058.t009","gold_sense_keys":["seem%2:39:01::"],"candidates":[{"index":1,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"},{"index":2,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval2.d001.s058.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"seem%2:39:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":941,"reasoning_output_tokens":920},"cost":{"currency":"USD","total_usd":0.030864999999999997,"input_uncached_usd":0.002635,"input_cached_usd":0.0,"output_usd":0.02823,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":18.30459420895204}
{"item_id":"senseval2.d001.s058.t010","gold_sense_keys":["develop%2:30:00::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":3,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":4,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":5,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":6,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":7,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":8,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":9,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":10,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":11,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":12,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":13,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":14,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":15,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":16,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"},{"index":17,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":18,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":19,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":20,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":21,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"develop%2:30:00::","call_ids":["senseval2.d001.s058.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"develop%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1284,"cached_input_tokens":0,"output_tokens":335,"reasoning_output_tokens":319},"cost":{"currency":"USD","total_usd":0.01647,"input_uncached_usd":0.00642,"input_cached_usd":0.0,"output_usd":0.01005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.511045707971789}
{"item_id":"senseval2.d001.s058.t011","gold_sense_keys":["stage%1:26:00::","stage%1:28:00::"],"candidates":[{"index":1,"sense_key":"stage%1:04:00::","synset_id":"stage.n.06"},{"index":2,"sense_key":"stage%1:26:00::","synset_id":"degree.n.02"},{"index":3,"sense_key":"stage%1:28:00::","synset_id":"phase.n.01"},{"index":4,"sense_key":"stage%1:06:00::","synset_id":"stage.n.03"},{"index":5,"sense_key":"stage%1:06:02::","synset_id":"stage.n.08"},{"index":6,"sense_key":"stage%1:15:00::","synset_id":"stage.n.07"},{"index":7,"sense_key":"stage%1:06:01::","synset_id":"stagecoach.n.01"},{"index":8,"sense_key":"stage%1:10:00::","synset_id":"stage.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"stage%1:28:00::","call_ids":["senseval2.d001.s058.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"stage%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":6092,"reasoning_output_tokens":6008},"cost":{"currency":"USD","total_usd":0.18638000000000002,"input_uncached_usd":0.0036200000000000004,"input_cached_usd":0.0,"output_usd":0.18276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":114.13096762506757}
{"item_id":"senseval2.d001.s059.t000","gold_sense_keys":["often%4:02:00::","often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":2,"sense_key":"often%4:02:02::","synset_id":"often.r.03"},{"index":3,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"often%4:02:02::","call_ids":["senseval2.d001.s059.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"often%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":105},"cost":{"currency":"USD","total_usd":0.005955,"input_uncached_usd":0.0023250000000000002,"input_cached_usd":0.0,"output_usd":0.00363,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8752143329475075}
{"item_id":"senseval2.d001.s059.t001","gold_sense_keys":["precede%2:42:00::"],"candidates":[{"index":1,"sense_key":"precede%2:38:00::","synset_id":"precede.v.04"},{"index":2,"sense_key":"precede%2:41:00::","synset_id":"precede.v.03"},{"index":3,"sense_key":"precede%2:42:01::","synset_id":"precede.v.02"},{"index":4,"sense_key":"precede%2:32:03::","synset_id":"precede.v.05"},{"index":5,"sense_key":"precede%2:42:00::","synset_id":"predate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"precede%2:42:00::","call_ids":["senseval2.d001.s059.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"precede%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":785,"reasoning_output_tokens":763},"cost":{"currency":"USD","total_usd":0.026445000000000003,"input_uncached_usd":0.0028950000000000004,"input_cached_usd":0.0,"output_usd":0.02355,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.447903749998659}
{"item_id":"senseval2.d001.s059.t002","gold_sense_keys":["development%1:22:01::"],"candidates":[{"index":1,"sense_key":"development%1:10:00::","synset_id":"development.n.09"},{"index":2,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":3,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":4,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":5,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":6,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":7,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":8,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":9,"sense_key":"development%1:04:01::","synset_id":"development.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"development%1:22:01::","call_ids":["senseval2.d001.s059.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"development%1:22:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":831,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019995,"input_uncached_usd":0.004155000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.550198124954477}
{"item_id":"senseval2.d001.s059.t003","gold_sense_keys":["polyp%1:26:00::"],"candidates":[{"index":1,"sense_key":"polyp%1:26:00::","synset_id":"polyp.n.01"},{"index":2,"sense_key":"polyp%1:05:00::","synset_id":"polyp.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"polyp%1:26:00::","call_ids":["senseval2.d001.s059.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"polyp%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.003940000000000001,"input_uncached_usd":0.00238,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.661711790948175}
{"item_id":"senseval2.d001.s059.t005","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":2,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":3,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":4,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":5,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":6,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":7,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":8,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":9,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":10,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":11,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":12,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":13,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":14,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":15,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":16,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":17,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":18,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":19,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"case%1:11:00::","call_ids":["senseval2.d001.s059.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"case%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1216,"cached_input_tokens":0,"output_tokens":242,"reasoning_output_tokens":227},"cost":{"currency":"USD","total_usd":0.013340000000000001,"input_uncached_usd":0.00608,"input_cached_usd":0.0,"output_usd":0.00726,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.708304000087082}
{"item_id":"senseval2.d001.s059.t006","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":2,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":3,"sense_key":"become%2:42:03::","synset_id":"become.v.04"},{"index":4,"sense_key":"become%2:30:00::","synset_id":"become.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"become%2:30:00::","call_ids":["senseval2.d001.s059.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"become%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":188,"reasoning_output_tokens":172},"cost":{"currency":"USD","total_usd":0.008235000000000001,"input_uncached_usd":0.002595,"input_cached_usd":0.0,"output_usd":0.00564,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.973003666033037}
{"item_id":"senseval2.d001.s059.t010","gold_sense_keys":["stage%1:26:00::","stage%1:28:00::"],"candidates":[{"index":1,"sense_key":"stage%1:15:00::","synset_id":"stage.n.07"},{"index":2,"sense_key":"stage%1:06:02::","synset_id":"stage.n.08"},{"index":3,"sense_key":"stage%1:28:00::","synset_id":"phase.n.01"},{"index":4,"sense_key":"stage%1:06:01::","synset_id":"stagecoach.n.01"},{"index":5,"sense_key":"stage%1:04:00::","synset_id":"stage.n.06"},{"index":6,"sense_key":"stage%1:10:00::","synset_id":"stage.n.04"},{"index":7,"sense_key":"stage%1:26:00::","synset_id":"degree.n.02"},{"index":8,"sense_key":"stage%1:06:00::","synset_id":"stage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"stage%1:26:00::","call_ids":["senseval2.d001.s059.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"stage%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":736,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01952,"input_uncached_usd":0.00368,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.109285125043243}
{"item_id":"senseval2.d001.s059.t011","gold_sense_keys":["progress%2:38:00::"],"candidates":[{"index":1,"sense_key":"progress%2:30:00::","synset_id":"progress.v.01"},{"index":2,"sense_key":"progress%2:38:00::","synset_id":"advance.v.01"},{"index":3,"sense_key":"progress%2:30:01::","synset_id":"build_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"progress%2:38:00::","call_ids":["senseval2.d001.s059.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"progress%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":101},"cost":{"currency":"USD","total_usd":0.006070000000000001,"input_uncached_usd":0.00256,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2944636669708416}
{"item_id":"senseval2.d001.s059.t012","gold_sense_keys":["less%4:02:00::"],"candidates":[{"index":1,"sense_key":"less%4:02:00::","synset_id":"less.r.01"},{"index":2,"sense_key":"less%4:02:01::","synset_id":"less.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"less%4:02:00::","call_ids":["senseval2.d001.s059.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"less%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.0049250000000000006,"input_uncached_usd":0.0021650000000000003,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4946067080600187}
{"item_id":"senseval2.d001.s059.t013","gold_sense_keys":["severe%5:00:00:critical:03","severe%5:00:00:intense:00","severe%5:00:01:bad:00"],"candidates":[{"index":1,"sense_key":"severe%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":2,"sense_key":"severe%5:00:00:intense:00","synset_id":"severe.s.01"},{"index":3,"sense_key":"severe%5:00:01:bad:00","synset_id":"severe.s.06"},{"index":4,"sense_key":"severe%5:00:02:nonindulgent:00","synset_id":"severe.s.04"},{"index":5,"sense_key":"severe%5:00:00:critical:03","synset_id":"dangerous.s.01"},{"index":6,"sense_key":"severe%5:00:00:plain:01","synset_id":"austere.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"severe%5:00:00:critical:03","call_ids":["senseval2.d001.s059.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"severe%5:00:00:critical:03","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":646,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01907,"input_uncached_usd":0.0032300000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.029710790957324}
{"item_id":"senseval2.d001.s059.t014","gold_sense_keys":["deadly%5:00:01:fatal:00"],"candidates":[{"index":1,"sense_key":"deadly%5:00:02:fatal:00","synset_id":"deadly.s.02"},{"index":2,"sense_key":"deadly%5:00:00:noxious:00","synset_id":"baneful.s.01"},{"index":3,"sense_key":"deadly%5:00:00:virulent:00","synset_id":"deadly.s.06"},{"index":4,"sense_key":"deadly%5:00:00:unpardonable:00","synset_id":"deadly.s.04"},{"index":5,"sense_key":"deadly%5:00:00:toxic:00","synset_id":"deadly.s.03"},{"index":6,"sense_key":"deadly%5:00:01:fatal:00","synset_id":"deadly.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"deadly%5:00:01:fatal:00","call_ids":["senseval2.d001.s059.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"deadly%5:00:01:fatal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":215,"reasoning_output_tokens":199},"cost":{"currency":"USD","total_usd":0.009485,"input_uncached_usd":0.0030350000000000004,"input_cached_usd":0.0,"output_usd":0.00645,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.565684915985912}
{"item_id":"senseval2.d001.s059.t015","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":2,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":3,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"},{"index":4,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s059.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":107},"cost":{"currency":"USD","total_usd":0.006325000000000001,"input_uncached_usd":0.0026650000000000003,"input_cached_usd":0.0,"output_usd":0.00366,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8752702500205487}
{"item_id":"senseval2.d001.s059.t016","gold_sense_keys":["damage%1:11:00::"],"candidates":[{"index":1,"sense_key":"damage%1:11:00::","synset_id":"damage.n.01"},{"index":2,"sense_key":"damage%1:11:01::","synset_id":"damage.n.02"},{"index":3,"sense_key":"damage%1:04:00::","synset_id":"damage.n.03"},{"index":4,"sense_key":"damage%1:21:00::","synset_id":"price.n.02"},{"index":5,"sense_key":"damage%1:04:01::","synset_id":"wrong.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%1:11:00::","call_ids":["senseval2.d001.s059.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":88},"cost":{"currency":"USD","total_usd":0.005725,"input_uncached_usd":0.002635,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4501223750412464}
{"item_id":"senseval2.d001.s059.t017","gold_sense_keys":["occur%2:30:00::"],"candidates":[{"index":1,"sense_key":"occur%2:30:00::","synset_id":"happen.v.01"},{"index":2,"sense_key":"occur%2:42:03::","synset_id":"occur.v.03"},{"index":3,"sense_key":"occur%2:31:00::","synset_id":"occur.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"occur%2:30:00::","call_ids":["senseval2.d001.s059.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"occur%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.005995,"input_uncached_usd":0.0024850000000000002,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9701500829542056}
{"item_id":"senseval2.d001.s060.t001","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"},{"index":2,"sense_key":"student%1:18:00::","synset_id":"student.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d001.s060.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.004645,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.120259708026424}
{"item_id":"senseval2.d001.s060.t002","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":2,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":3,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":4,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"},{"index":9,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":10,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"begin%2:30:00::","call_ids":["senseval2.d001.s060.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"begin%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":890,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02029,"input_uncached_usd":0.00445,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.184355875011533}
{"item_id":"senseval2.d001.s060.t003","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["senseval2.d001.s060.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.004355,"input_uncached_usd":0.002285,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.006861582980491}
{"item_id":"senseval2.d001.s060.t004","gold_sense_keys":["tedious%5:00:00:uninteresting:00"],"candidates":[{"index":1,"sense_key":"tedious%5:00:00:prolix:00","synset_id":"long-winded.s.01"},{"index":2,"sense_key":"tedious%5:00:00:uninteresting:00","synset_id":"boring.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tedious%5:00:00:uninteresting:00","call_ids":["senseval2.d001.s060.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tedious%5:00:00:uninteresting:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.004995,"input_uncached_usd":0.002625,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7336037500062957}
{"item_id":"senseval2.d001.s060.t005","gold_sense_keys":["often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:02::","synset_id":"often.r.03"},{"index":3,"sense_key":"often%4:02:01::","synset_id":"much.r.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"often%4:02:02::","call_ids":["senseval2.d001.s060.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"often%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0183,"input_uncached_usd":0.0024600000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.44811437500175}
{"item_id":"senseval2.d001.s060.t009","gold_sense_keys":["search%2:35:00::","search%2:39:00::"],"candidates":[{"index":1,"sense_key":"search%2:39:00::","synset_id":"search.v.02"},{"index":2,"sense_key":"search%2:31:00::","synset_id":"research.v.02"},{"index":3,"sense_key":"search%2:35:00::","synset_id":"search.v.01"},{"index":4,"sense_key":"search%2:35:01::","synset_id":"search.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"search%2:35:00::","call_ids":["senseval2.d001.s060.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"search%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":580,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":122},"cost":{"currency":"USD","total_usd":0.007040000000000001,"input_uncached_usd":0.0029000000000000002,"input_cached_usd":0.0,"output_usd":0.0041400000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0372959160013124}
{"item_id":"senseval2.d001.s060.t010","gold_sense_keys":["sign%1:26:00::"],"candidates":[{"index":1,"sense_key":"sign%1:10:01::","synset_id":"sign.n.09"},{"index":2,"sense_key":"sign%1:10:03::","synset_id":"signal.n.01"},{"index":3,"sense_key":"sign%1:06:01::","synset_id":"signboard.n.01"},{"index":4,"sense_key":"sign%1:10:04::","synset_id":"sign.n.11"},{"index":5,"sense_key":"sign%1:10:05::","synset_id":"sign.n.01"},{"index":6,"sense_key":"sign%1:11:00::","synset_id":"augury.n.01"},{"index":7,"sense_key":"sign%1:24:00::","synset_id":"polarity.n.02"},{"index":8,"sense_key":"sign%1:10:06::","synset_id":"sign.n.10"},{"index":9,"sense_key":"sign%1:10:00::","synset_id":"sign.n.02"},{"index":10,"sense_key":"sign%1:26:00::","synset_id":"sign.n.06"},{"index":11,"sense_key":"sign%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"sign%1:10:05::","call_ids":["senseval2.d001.s060.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"sign%1:10:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":863,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":141},"cost":{"currency":"USD","total_usd":0.008995,"input_uncached_usd":0.004315,"input_cached_usd":0.0,"output_usd":0.00468,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.852589915972203}
{"item_id":"senseval2.d001.s060.t011","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":2,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":3,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":4,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s060.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":117},"cost":{"currency":"USD","total_usd":0.006790000000000001,"input_uncached_usd":0.0028000000000000004,"input_cached_usd":0.0,"output_usd":0.0039900000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3874661249574274}
{"item_id":"senseval2.d001.s060.t012","gold_sense_keys":["damage%1:11:00::"],"candidates":[{"index":1,"sense_key":"damage%1:21:00::","synset_id":"price.n.02"},{"index":2,"sense_key":"damage%1:04:00::","synset_id":"damage.n.03"},{"index":3,"sense_key":"damage%1:11:00::","synset_id":"damage.n.01"},{"index":4,"sense_key":"damage%1:11:01::","synset_id":"damage.n.02"},{"index":5,"sense_key":"damage%1:04:01::","synset_id":"wrong.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"damage%1:11:00::","call_ids":["senseval2.d001.s060.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"damage%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":88},"cost":{"currency":"USD","total_usd":0.005885,"input_uncached_usd":0.002765,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.961731916060671}
{"item_id":"senseval2.d001.s061.t000","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":2,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":3,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":4,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":5,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":6,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"},{"index":7,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":8,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":9,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":10,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"begin%2:30:00::","call_ids":["senseval2.d001.s061.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"begin%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":857,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":111},"cost":{"currency":"USD","total_usd":0.008095000000000001,"input_uncached_usd":0.004285000000000001,"input_cached_usd":0.0,"output_usd":0.00381,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6298775419127196}
{"item_id":"senseval2.d001.s061.t001","gold_sense_keys":["confusing%5:00:00:unclear:00"],"candidates":[{"index":1,"sense_key":"confusing%5:00:00:unclear:00","synset_id":"confusing.s.02"},{"index":2,"sense_key":"confusing%5:00:00:disorienting:00","synset_id":"confusing.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"confusing%5:00:00:unclear:00","call_ids":["senseval2.d001.s061.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"confusing%5:00:00:unclear:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.004665000000000001,"input_uncached_usd":0.002295,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.075420125038363}
{"item_id":"senseval2.d001.s061.t002","gold_sense_keys":["variety%1:07:00::","variety%1:14:01::"],"candidates":[{"index":1,"sense_key":"variety%1:09:01::","synset_id":"kind.n.01"},{"index":2,"sense_key":"variety%1:07:01::","synset_id":"variety.n.06"},{"index":3,"sense_key":"variety%1:07:00::","synset_id":"diverseness.n.01"},{"index":4,"sense_key":"variety%1:14:00::","synset_id":"variety.n.03"},{"index":5,"sense_key":"variety%1:10:00::","synset_id":"variety_show.n.01"},{"index":6,"sense_key":"variety%1:14:01::","synset_id":"assortment.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"variety%1:14:01::","call_ids":["senseval2.d001.s061.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"variety%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":458,"reasoning_output_tokens":442},"cost":{"currency":"USD","total_usd":0.017,"input_uncached_usd":0.0032600000000000003,"input_cached_usd":0.0,"output_usd":0.01374,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.676046708947979}
{"item_id":"senseval2.d001.s061.t003","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":2,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"},{"index":3,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":4,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s061.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":112},"cost":{"currency":"USD","total_usd":0.006475,"input_uncached_usd":0.002635,"input_cached_usd":0.0,"output_usd":0.00384,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9445332910399884}
{"item_id":"senseval2.d001.s061.t004","gold_sense_keys":["exist%2:42:00::"],"candidates":[{"index":1,"sense_key":"exist%2:42:00::","synset_id":"exist.v.01"},{"index":2,"sense_key":"exist%2:42:01::","synset_id":"exist.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"exist%2:42:00::","call_ids":["senseval2.d001.s061.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"exist%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":438,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.00432,"input_uncached_usd":0.00219,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1368062919937074}
{"item_id":"senseval2.d001.s061.t005","gold_sense_keys":["only%4:02:01::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:05::","synset_id":"only.r.07"},{"index":3,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":4,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":5,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:04::","synset_id":"only.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"only%4:02:01::","call_ids":["senseval2.d001.s061.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"only%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.00545,"input_uncached_usd":0.0031400000000000004,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7781063340371475}
{"item_id":"senseval2.d001.s061.t006","gold_sense_keys":["polyp%1:26:00::"],"candidates":[{"index":1,"sense_key":"polyp%1:26:00::","synset_id":"polyp.n.01"},{"index":2,"sense_key":"polyp%1:05:00::","synset_id":"polyp.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"polyp%1:26:00::","call_ids":["senseval2.d001.s061.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"polyp%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004115000000000001,"input_uncached_usd":0.0023450000000000003,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8803939169738442}
{"item_id":"senseval2.d001.s061.t008","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":2,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":3,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":4,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":5,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":6,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":7,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s061.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.005175000000000001,"input_uncached_usd":0.003105,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.334113833028823}
{"item_id":"senseval2.d001.s061.t010","gold_sense_keys":["polyp%1:26:00::"],"candidates":[{"index":1,"sense_key":"polyp%1:05:00::","synset_id":"polyp.n.02"},{"index":2,"sense_key":"polyp%1:26:00::","synset_id":"polyp.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"polyp%1:26:00::","call_ids":["senseval2.d001.s061.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"polyp%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.00385,"input_uncached_usd":0.00235,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7100830839481205}
{"item_id":"senseval2.d001.s061.t012","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":4,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":5,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":6,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":7,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s061.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":41,"reasoning_output_tokens":25},"cost":{"currency":"USD","total_usd":0.0043300000000000005,"input_uncached_usd":0.0031000000000000003,"input_cached_usd":0.0,"output_usd":0.00123,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.234244583058171}
{"item_id":"senseval2.d001.s062.t001","gold_sense_keys":["coherent%3:00:00::"],"candidates":[{"index":1,"sense_key":"coherent%3:00:01::","synset_id":"coherent.a.03"},{"index":2,"sense_key":"coherent%5:00:00:adhesive:00","synset_id":"coherent.s.02"},{"index":3,"sense_key":"coherent%3:00:00::","synset_id":"coherent.a.01"},{"index":4,"sense_key":"coherent%5:00:00:rational:00","synset_id":"coherent.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"coherent%3:00:00::","call_ids":["senseval2.d001.s062.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"coherent%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.00419,"input_uncached_usd":0.00254,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6830152499023825}
{"item_id":"senseval2.d001.s062.t002","gold_sense_keys":["picture%1:26:00::"],"candidates":[{"index":1,"sense_key":"picture%1:09:00::","synset_id":"mental_picture.n.01"},{"index":2,"sense_key":"picture%1:10:02::","synset_id":"movie.n.01"},{"index":3,"sense_key":"picture%1:06:00::","synset_id":"picture.n.01"},{"index":4,"sense_key":"picture%1:10:01::","synset_id":"video.n.01"},{"index":5,"sense_key":"picture%1:06:02::","synset_id":"photograph.n.01"},{"index":6,"sense_key":"picture%1:26:00::","synset_id":"picture.n.04"},{"index":7,"sense_key":"picture%1:09:01::","synset_id":"picture.n.09"},{"index":8,"sense_key":"picture%1:06:01::","synset_id":"painting.n.01"},{"index":9,"sense_key":"picture%1:10:00::","synset_id":"picture.n.05"},{"index":10,"sense_key":"picture%1:10:03::","synset_id":"word_picture.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"picture%1:09:00::","call_ids":["senseval2.d001.s062.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"picture%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":871,"cached_input_tokens":0,"output_tokens":196,"reasoning_output_tokens":180},"cost":{"currency":"USD","total_usd":0.010235000000000001,"input_uncached_usd":0.004355,"input_cached_usd":0.0,"output_usd":0.00588,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.89613091701176}
{"item_id":"senseval2.d001.s062.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":2,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":3,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":4,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s062.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.004365000000000001,"input_uncached_usd":0.0026850000000000003,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7352449999889359}
{"item_id":"senseval2.d001.s062.t004","gold_sense_keys":["development%1:22:02::"],"candidates":[{"index":1,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":2,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":3,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":4,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":5,"sense_key":"development%1:10:00::","synset_id":"development.n.09"},{"index":6,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":7,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":8,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":9,"sense_key":"development%1:22:00::","synset_id":"development.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"development%1:22:02::","call_ids":["senseval2.d001.s062.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"development%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":813,"cached_input_tokens":0,"output_tokens":214,"reasoning_output_tokens":198},"cost":{"currency":"USD","total_usd":0.010485000000000001,"input_uncached_usd":0.004065,"input_cached_usd":0.0,"output_usd":0.00642,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.840186499990523}
{"item_id":"senseval2.d001.s062.t005","gold_sense_keys":["emerge%2:30:01::"],"candidates":[{"index":1,"sense_key":"emerge%2:42:00::","synset_id":"come_forth.v.02"},{"index":2,"sense_key":"emerge%2:30:02::","synset_id":"issue.v.04"},{"index":3,"sense_key":"emerge%2:38:00::","synset_id":"emerge.v.04"},{"index":4,"sense_key":"emerge%2:30:00::","synset_id":"emerge.v.01"},{"index":5,"sense_key":"emerge%2:30:01::","synset_id":"emerge.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"emerge%2:30:01::","call_ids":["senseval2.d001.s062.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"emerge%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0043300000000000005,"input_uncached_usd":0.00265,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6462736249668524}
{"item_id":"senseval2.d001.s063.t000","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":2,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"},{"index":3,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":4,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s063.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.0047150000000000004,"input_uncached_usd":0.002405,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7323572500608861}
{"item_id":"senseval2.d001.s063.t001","gold_sense_keys":["certain%5:00:00:definite:00"],"candidates":[{"index":1,"sense_key":"certain%3:00:02::","synset_id":"certain.a.02"},{"index":2,"sense_key":"certain%5:00:00:reliable:00","synset_id":"certain.s.02"},{"index":3,"sense_key":"certain%5:00:00:definite:00","synset_id":"certain.s.01"},{"index":4,"sense_key":"certain%5:00:00:careful:00","synset_id":"certain.s.03"},{"index":5,"sense_key":"certain%3:00:03::","synset_id":"certain.a.04"},{"index":6,"sense_key":"certain%3:00:05::","synset_id":"sealed.a.01"},{"index":7,"sense_key":"certain%3:00:01::","synset_id":"certain.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"certain%5:00:00:definite:00","call_ids":["senseval2.d001.s063.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"certain%5:00:00:definite:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.005305000000000001,"input_uncached_usd":0.003265,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4783639169763774}
{"item_id":"senseval2.d001.s063.t003","gold_sense_keys":["knock_out%2:30:01::"],"candidates":[{"index":1,"sense_key":"knock_out%2:35:00::","synset_id":"knock_cold.v.01"},{"index":2,"sense_key":"knock_out%2:30:00::","synset_id":"knock_out.v.03"},{"index":3,"sense_key":"knock_out%2:37:00::","synset_id":"knock_out.v.04"},{"index":4,"sense_key":"knock_out%2:30:01::","synset_id":"knock_out.v.01"},{"index":5,"sense_key":"knock_out%2:30:02::","synset_id":"knock_out.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"knock_out%2:30:01::","call_ids":["senseval2.d001.s063.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"knock_out%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.005325,"input_uncached_usd":0.002565,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3189598330063745}
{"item_id":"senseval2.d001.s063.t004","gold_sense_keys":["polyp%1:26:00::"],"candidates":[{"index":1,"sense_key":"polyp%1:26:00::","synset_id":"polyp.n.01"},{"index":2,"sense_key":"polyp%1:05:00::","synset_id":"polyp.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"polyp%1:26:00::","call_ids":["senseval2.d001.s063.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"polyp%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.003885,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3953515000175685}
{"item_id":"senseval2.d001.s063.t005","gold_sense_keys":["develop%2:30:00::","develop%2:42:00::"],"candidates":[{"index":1,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":2,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":3,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":4,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":5,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":6,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":7,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":8,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":9,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":12,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":13,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":14,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":15,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":16,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":17,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"},{"index":18,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":19,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":20,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":21,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"develop%2:42:00::","call_ids":["senseval2.d001.s063.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"develop%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1271,"cached_input_tokens":0,"output_tokens":4835,"reasoning_output_tokens":4768},"cost":{"currency":"USD","total_usd":0.151405,"input_uncached_usd":0.006355,"input_cached_usd":0.0,"output_usd":0.14505,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":130.91631800006144}
{"item_id":"senseval2.d001.s064.t000","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"},{"index":2,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":3,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":4,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s064.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.0049700000000000005,"input_uncached_usd":0.00236,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.543881291989237}
{"item_id":"senseval2.d001.s064.t001","gold_sense_keys":["second%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"second%3:00:00::","synset_id":"second.a.02"},{"index":2,"sense_key":"second%5:00:00:ordinal:00","synset_id":"second.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"second%5:00:00:ordinal:00","call_ids":["senseval2.d001.s064.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"second%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.00355,"input_uncached_usd":0.00205,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.676841750042513}
{"item_id":"senseval2.d001.s064.t003","gold_sense_keys":["then%4:02:00::"],"candidates":[{"index":1,"sense_key":"then%4:02:01::","synset_id":"then.r.03"},{"index":2,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":3,"sense_key":"then%4:02:00::","synset_id":"then.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"then%4:02:00::","call_ids":["senseval2.d001.s064.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"then%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":116},"cost":{"currency":"USD","total_usd":0.006185,"input_uncached_usd":0.002255,"input_cached_usd":0.0,"output_usd":0.00393,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5890565420268103}
{"item_id":"senseval2.d001.s064.t004","gold_sense_keys":["delete%2:35:00::"],"candidates":[{"index":1,"sense_key":"delete%2:30:00::","synset_id":"edit.v.04"},{"index":2,"sense_key":"delete%2:32:00::","synset_id":"erase.v.03"},{"index":3,"sense_key":"delete%2:35:00::","synset_id":"delete.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"delete%2:35:00::","call_ids":["senseval2.d001.s064.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"delete%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":129},"cost":{"currency":"USD","total_usd":0.00651,"input_uncached_usd":0.00216,"input_cached_usd":0.0,"output_usd":0.00435,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.260509208892472}
{"item_id":"senseval2.d001.s064.t005","gold_sense_keys":["polyp%1:26:00::"],"candidates":[{"index":1,"sense_key":"polyp%1:26:00::","synset_id":"polyp.n.01"},{"index":2,"sense_key":"polyp%1:05:00::","synset_id":"polyp.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"polyp%1:26:00::","call_ids":["senseval2.d001.s064.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"polyp%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.0037500000000000003,"input_uncached_usd":0.00222,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5684862500056624}
{"item_id":"senseval2.d001.s064.t007","gold_sense_keys":["malignancy%1:26:00::"],"candidates":[{"index":1,"sense_key":"malignancy%1:07:00::","synset_id":"malignity.n.02"},{"index":2,"sense_key":"malignancy%1:26:00::","synset_id":"malignancy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"malignancy%1:26:00::","call_ids":["senseval2.d001.s064.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"malignancy%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":39,"reasoning_output_tokens":23},"cost":{"currency":"USD","total_usd":0.0031850000000000003,"input_uncached_usd":0.0020150000000000003,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4770263750106096}
{"item_id":"senseval2.d001.s065.t000","gold_sense_keys":["clear%3:00:00::"],"candidates":[{"index":1,"sense_key":"clear%5:00:00:untroubled:00","synset_id":"clear.s.04"},{"index":2,"sense_key":"clear%5:00:00:unobstructed:00","synset_id":"clear.s.02"},{"index":3,"sense_key":"clear%5:00:00:legible:00","synset_id":"clear.s.11"},{"index":4,"sense_key":"clear%5:00:01:pure:02","synset_id":"clean.s.02"},{"index":5,"sense_key":"clear%3:00:00::","synset_id":"clear.a.01"},{"index":6,"sense_key":"clear%5:00:02:free:00","synset_id":"clear.s.03"},{"index":7,"sense_key":"clear%5:00:00:unencumbered:00","synset_id":"clear.s.06"},{"index":8,"sense_key":"clear%3:00:02::","synset_id":"clear.a.04"},{"index":9,"sense_key":"clear%3:00:03::","synset_id":"clear.a.11"},{"index":10,"sense_key":"clear%5:00:00:discerning:00","synset_id":"clear.s.13"},{"index":11,"sense_key":"clear%5:00:00:clearheaded:00","synset_id":"clear.s.01"},{"index":12,"sense_key":"clear%5:00:00:innocent:00","synset_id":"absolved.s.01"},{"index":13,"sense_key":"clear%5:00:00:net:00","synset_id":"clear.s.10"},{"index":14,"sense_key":"clear%5:00:00:unqualified:02","synset_id":"clean.s.01"},{"index":15,"sense_key":"clear%5:00:00:perfect:00","synset_id":"clear.s.09"},{"index":16,"sense_key":"clear%3:00:04::","synset_id":"well-defined.a.02"},{"index":17,"sense_key":"clear%5:00:00:distinct:00","synset_id":"clear.s.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"clear%3:00:00::","call_ids":["senseval2.d001.s065.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"clear%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1037,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.007105,"input_uncached_usd":0.005185,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4037935830419883}
{"item_id":"senseval2.d001.s065.t003","gold_sense_keys":["damage%2:30:00::"],"candidates":[{"index":1,"sense_key":"damage%2:30:00::","synset_id":"damage.v.01"},{"index":2,"sense_key":"damage%2:30:01::","synset_id":"damage.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%2:30:00::","call_ids":["senseval2.d001.s065.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":359,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":123},"cost":{"currency":"USD","total_usd":0.005965,"input_uncached_usd":0.0017950000000000002,"input_cached_usd":0.0,"output_usd":0.00417,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.147366250050254}
{"item_id":"senseval2.d001.s065.t006","gold_sense_keys":["develop%2:30:00::","develop%2:42:00::"],"candidates":[{"index":1,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":2,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":3,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":4,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":5,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":6,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":7,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":8,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":9,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":10,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":11,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":12,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":15,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":16,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"},{"index":17,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":18,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":19,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":20,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":21,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"develop%2:42:00::","call_ids":["senseval2.d001.s065.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"develop%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1227,"cached_input_tokens":0,"output_tokens":6292,"reasoning_output_tokens":6204},"cost":{"currency":"USD","total_usd":0.194895,"input_uncached_usd":0.006135000000000001,"input_cached_usd":0.0,"output_usd":0.18876,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":139.25341220898554}
{"item_id":"senseval2.d001.s066.t000","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":2,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":5,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:11:00::","synset_id":"report.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"report%1:10:03::","call_ids":["senseval2.d001.s066.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0048000000000000004,"input_uncached_usd":0.0030900000000000003,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6187851669965312}
{"item_id":"senseval2.d001.s066.t001","gold_sense_keys":["galvanize%2:37:00::"],"candidates":[{"index":1,"sense_key":"galvanize%2:37:00::","synset_id":"startle.v.01"},{"index":2,"sense_key":"galvanize%2:35:00::","synset_id":"galvanize.v.02"},{"index":3,"sense_key":"galvanize%2:29:02::","synset_id":"galvanize.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"galvanize%2:37:00::","call_ids":["senseval2.d001.s066.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"galvanize%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":408,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.00381,"input_uncached_usd":0.00204,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6701424159109592}
{"item_id":"senseval2.d001.s066.t002","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":2,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s066.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004135,"input_uncached_usd":0.0023350000000000003,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.766948875039816}
{"item_id":"senseval2.d001.s067.t000","gold_sense_keys":["confirming%5:00:00:supportive:00"],"candidates":[{"index":1,"sense_key":"confirming%3:00:02::","synset_id":"positive.a.04"},{"index":2,"sense_key":"confirming%5:00:00:supportive:00","synset_id":"collateral.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"confirming%5:00:00:supportive:00","call_ids":["senseval2.d001.s067.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"confirming%5:00:00:supportive:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":385,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.003845,"input_uncached_usd":0.001925,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7303893339121714}
{"item_id":"senseval2.d001.s067.t001","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":2,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"},{"index":3,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval2.d001.s067.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":411,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.005145,"input_uncached_usd":0.002055,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4791567919310182}
{"item_id":"senseval2.d001.s067.t002","gold_sense_keys":["need%2:42:00::"],"candidates":[{"index":1,"sense_key":"need%2:34:00::","synset_id":"want.v.02"},{"index":2,"sense_key":"need%2:42:00::","synset_id":"necessitate.v.01"},{"index":3,"sense_key":"need%2:34:01::","synset_id":"need.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"need%2:34:01::","call_ids":["senseval2.d001.s067.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"need%2:34:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017915,"input_uncached_usd":0.002075,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.24792858294677}
{"item_id":"senseval2.d001.s067.t004","gold_sense_keys":["loss%1:04:00::"],"candidates":[{"index":1,"sense_key":"loss%1:21:01::","synset_id":"loss.n.01"},{"index":2,"sense_key":"loss%1:04:00::","synset_id":"loss.n.03"},{"index":3,"sense_key":"loss%1:22:00::","synset_id":"loss.n.02"},{"index":4,"sense_key":"loss%1:07:00::","synset_id":"loss.n.04"},{"index":5,"sense_key":"loss%1:11:02::","synset_id":"passing.n.02"},{"index":6,"sense_key":"loss%1:21:00::","synset_id":"loss.n.06"},{"index":7,"sense_key":"loss%1:11:00::","synset_id":"personnel_casualty.n.01"},{"index":8,"sense_key":"loss%1:11:01::","synset_id":"loss.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"loss%1:04:00::","call_ids":["senseval2.d001.s067.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"loss%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":497,"reasoning_output_tokens":481},"cost":{"currency":"USD","total_usd":0.018,"input_uncached_usd":0.0030900000000000003,"input_cached_usd":0.0,"output_usd":0.01491,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":16.133148750057444}
{"item_id":"senseval2.d001.s067.t005","gold_sense_keys":["critical%5:00:00:indispensable:00"],"candidates":[{"index":1,"sense_key":"critical%3:00:04::","synset_id":"critical.a.02"},{"index":2,"sense_key":"critical%3:01:00::","synset_id":"critical.a.07"},{"index":3,"sense_key":"critical%3:00:02::","synset_id":"critical.a.03"},{"index":4,"sense_key":"critical%5:00:00:crucial:00","synset_id":"critical.s.02"},{"index":5,"sense_key":"critical%3:00:01::","synset_id":"critical.a.01"},{"index":6,"sense_key":"critical%5:00:00:indispensable:00","synset_id":"critical.s.01"},{"index":7,"sense_key":"critical%3:00:03::","synset_id":"critical.a.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"critical%5:00:00:indispensable:00","call_ids":["senseval2.d001.s067.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"critical%5:00:00:indispensable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.005755,"input_uncached_usd":0.002935,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1329917500261217}
{"item_id":"senseval2.d001.s067.t006","gold_sense_keys":["development%1:22:02::"],"candidates":[{"index":1,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":2,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":3,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":4,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":5,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":6,"sense_key":"development%1:10:00::","synset_id":"development.n.09"},{"index":7,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":8,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":9,"sense_key":"development%1:26:00::","synset_id":"development.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"development%1:22:02::","call_ids":["senseval2.d001.s067.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"development%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.03521,"input_uncached_usd":0.0038300000000000005,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":27.02306824992411}
{"item_id":"senseval2.d001.s067.t007","gold_sense_keys":["common%3:00:01::","common%5:00:00:familiar:02"],"candidates":[{"index":1,"sense_key":"common%5:00:00:unrefined:01","synset_id":"coarse.s.01"},{"index":2,"sense_key":"common%3:00:02::","synset_id":"common.a.01"},{"index":3,"sense_key":"common%5:00:00:ordinary:00","synset_id":"common.s.07"},{"index":4,"sense_key":"common%5:00:00:lowborn:00","synset_id":"common.s.04"},{"index":5,"sense_key":"common%5:00:00:shared:00","synset_id":"common.s.01"},{"index":6,"sense_key":"common%5:00:00:familiar:02","synset_id":"common.s.02"},{"index":7,"sense_key":"common%3:00:01::","synset_id":"common.a.02"},{"index":8,"sense_key":"common%5:00:00:inferior:02","synset_id":"coarse.s.02"},{"index":9,"sense_key":"common%5:00:00:informal:02","synset_id":"common.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"common%5:00:00:familiar:02","call_ids":["senseval2.d001.s067.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"common%5:00:00:familiar:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":390,"reasoning_output_tokens":374},"cost":{"currency":"USD","total_usd":0.01535,"input_uncached_usd":0.0036500000000000005,"input_cached_usd":0.0,"output_usd":0.0117,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.85545854200609}
{"item_id":"senseval2.d001.s067.t009","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":2,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":3,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":4,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":5,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":6,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":7,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":8,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":9,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:13::","synset_id":"say.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d001.s067.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":1937,"reasoning_output_tokens":1903},"cost":{"currency":"USD","total_usd":0.06196,"input_uncached_usd":0.00385,"input_cached_usd":0.0,"output_usd":0.05811,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":31.766658749897033}
{"item_id":"senseval2.d001.s068.t000","gold_sense_keys":["yet%4:02:02::"],"candidates":[{"index":1,"sense_key":"yet%4:02:00::","synset_id":"yet.r.04"},{"index":2,"sense_key":"yet%4:02:03::","synset_id":"even.r.03"},{"index":3,"sense_key":"yet%4:02:02::","synset_id":"yet.r.01"},{"index":4,"sense_key":"yet%4:02:06::","synset_id":"however.r.01"},{"index":5,"sense_key":"yet%4:02:04::","synset_id":"so_far.r.01"},{"index":6,"sense_key":"yet%4:02:05::","synset_id":"so_far.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"yet%4:02:02::","call_ids":["senseval2.d001.s068.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"yet%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":175,"reasoning_output_tokens":160},"cost":{"currency":"USD","total_usd":0.008375,"input_uncached_usd":0.003125,"input_cached_usd":0.0,"output_usd":0.00525,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.518210707930848}
{"item_id":"senseval2.d001.s068.t001","gold_sense_keys":["nail%2:40:00::"],"candidates":[{"index":1,"sense_key":"nail%2:40:00::","synset_id":"pinpoint.v.01"},{"index":2,"sense_key":"nail%2:35:00::","synset_id":"nail.v.01"},{"index":3,"sense_key":"nail%2:35:02::","synset_id":"smash.v.01"},{"index":4,"sense_key":"nail%2:33:00::","synset_id":"complete.v.04"},{"index":5,"sense_key":"nail%2:41:00::","synset_id":"breeze_through.v.01"},{"index":6,"sense_key":"nail%2:35:01::","synset_id":"collar.v.01"},{"index":7,"sense_key":"nail%2:41:01::","synset_id":"nail_down.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nail%2:40:00::","call_ids":["senseval2.d001.s068.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nail%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.004645,"input_uncached_usd":0.0028450000000000003,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8548325420124456}
{"item_id":"senseval2.d001.s068.t002","gold_sense_keys":["identity%1:09:00::"],"candidates":[{"index":1,"sense_key":"identity%1:07:00::","synset_id":"identity.n.01"},{"index":2,"sense_key":"identity%1:09:00::","synset_id":"identity.n.02"},{"index":3,"sense_key":"identity%1:07:02::","synset_id":"identity.n.04"},{"index":4,"sense_key":"identity%1:24:01::","synset_id":"identity.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"identity%1:09:00::","call_ids":["senseval2.d001.s068.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"identity%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.004785000000000001,"input_uncached_usd":0.0023850000000000004,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.119876457960345}
{"item_id":"senseval2.d001.s068.t004","gold_sense_keys":["damage%2:30:00::"],"candidates":[{"index":1,"sense_key":"damage%2:30:01::","synset_id":"damage.v.02"},{"index":2,"sense_key":"damage%2:30:00::","synset_id":"damage.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"damage%2:30:00::","call_ids":["senseval2.d001.s068.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"damage%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":356,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.0049,"input_uncached_usd":0.0017800000000000001,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.604460958042182}
{"item_id":"senseval2.d001.s068.t005","gold_sense_keys":["flip%2:30:00::"],"candidates":[{"index":1,"sense_key":"flip%2:35:03::","synset_id":"flip.v.08"},{"index":2,"sense_key":"flip%2:31:00::","synset_id":"flip.v.09"},{"index":3,"sense_key":"flip%2:38:00::","synset_id":"flip.v.01"},{"index":4,"sense_key":"flip%2:35:01::","synset_id":"throw.v.06"},{"index":5,"sense_key":"flip%2:35:00::","synset_id":"flip.v.06"},{"index":6,"sense_key":"flip%2:30:00::","synset_id":"interchange.v.04"},{"index":7,"sense_key":"flip%2:38:01::","synset_id":"flip.v.05"},{"index":8,"sense_key":"flip%2:39:10::","synset_id":"flick.v.02"},{"index":9,"sense_key":"flip%2:35:05::","synset_id":"flip.v.07"},{"index":10,"sense_key":"flip%2:38:02::","synset_id":"flip.v.04"},{"index":11,"sense_key":"flip%2:30:13::","synset_id":"flip.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"flip%2:30:00::","call_ids":["senseval2.d001.s068.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"flip%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":752,"cached_input_tokens":0,"output_tokens":816,"reasoning_output_tokens":795},"cost":{"currency":"USD","total_usd":0.028239999999999998,"input_uncached_usd":0.0037600000000000003,"input_cached_usd":0.0,"output_usd":0.024480000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.499727083952166}
{"item_id":"senseval2.d001.s068.t006","gold_sense_keys":["colon%1:08:00::"],"candidates":[{"index":1,"sense_key":"colon%1:23:01::","synset_id":"colon.n.02"},{"index":2,"sense_key":"colon%1:23:00::","synset_id":"colon.n.03"},{"index":3,"sense_key":"colon%1:08:00::","synset_id":"colon.n.01"},{"index":4,"sense_key":"colon%1:15:00::","synset_id":"colon.n.04"},{"index":5,"sense_key":"colon%1:10:00::","synset_id":"colon.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"colon%1:08:00::","call_ids":["senseval2.d001.s068.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"colon%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.0038250000000000003,"input_uncached_usd":0.002415,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8214152499567717}
{"item_id":"senseval2.d001.s068.t007","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":2,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":3,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":4,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":5,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":6,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":7,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s068.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":558,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.0045000000000000005,"input_uncached_usd":0.0027900000000000004,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7593330830568448}
{"item_id":"senseval2.d001.s068.t008","gold_sense_keys":["full-blown%5:00:00:complete:00"],"candidates":[{"index":1,"sense_key":"full-blown%5:00:00:mature:01","synset_id":"full-blown.s.01"},{"index":2,"sense_key":"full-blown%5:00:00:complete:00","synset_id":"full-blown.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"full-blown%5:00:00:complete:00","call_ids":["senseval2.d001.s068.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"full-blown%5:00:00:complete:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":109},"cost":{"currency":"USD","total_usd":0.00557,"input_uncached_usd":0.00185,"input_cached_usd":0.0,"output_usd":0.00372,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.961628874996677}
{"item_id":"senseval2.d001.s068.t009","gold_sense_keys":["malignancy%1:26:00::"],"candidates":[{"index":1,"sense_key":"malignancy%1:07:00::","synset_id":"malignity.n.02"},{"index":2,"sense_key":"malignancy%1:26:00::","synset_id":"malignancy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"malignancy%1:26:00::","call_ids":["senseval2.d001.s068.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"malignancy%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":366,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.00345,"input_uncached_usd":0.0018300000000000002,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6244755000807345}
{"item_id":"senseval2.d001.s070.t000","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["senseval2.d001.s070.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":372,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.00408,"input_uncached_usd":0.00186,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6754187080077827}
{"item_id":"senseval2.d001.s070.t002","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":2,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":3,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":6,"sense_key":"use%2:34:00::","synset_id":"use.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d001.s070.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.00541,"input_uncached_usd":0.00289,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.592925333068706}
{"item_id":"senseval2.d001.s070.t005","gold_sense_keys":["down%4:02:00::"],"candidates":[{"index":1,"sense_key":"down%4:02:06::","synset_id":"down.r.04"},{"index":2,"sense_key":"down%4:02:03::","synset_id":"down.r.03"},{"index":3,"sense_key":"down%4:02:02::","synset_id":"down.r.02"},{"index":4,"sense_key":"down%4:02:04::","synset_id":"down.r.06"},{"index":5,"sense_key":"down%4:02:05::","synset_id":"down.r.05"},{"index":6,"sense_key":"down%4:02:00::","synset_id":"down.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"down%4:02:00::","call_ids":["senseval2.d001.s070.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"down%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":542,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01855,"input_uncached_usd":0.00271,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.423254667082801}
{"item_id":"senseval2.d001.s070.t006","gold_sense_keys":["length%1:07:00::"],"candidates":[{"index":1,"sense_key":"length%1:07:00::","synset_id":"length.n.01"},{"index":2,"sense_key":"length%1:06:00::","synset_id":"length.n.05"},{"index":3,"sense_key":"length%1:07:01::","synset_id":"duration.n.03"},{"index":4,"sense_key":"length%1:07:02::","synset_id":"length.n.03"},{"index":5,"sense_key":"length%1:07:03::","synset_id":"distance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"length%1:07:00::","call_ids":["senseval2.d001.s070.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"length%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":95},"cost":{"currency":"USD","total_usd":0.005935,"input_uncached_usd":0.002605,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.897417041938752}
{"item_id":"senseval2.d001.s070.t008","gold_sense_keys":["look_for%2:35:00::"],"candidates":[{"index":1,"sense_key":"look_for%2:35:00::","synset_id":"search.v.01"},{"index":2,"sense_key":"look_for%2:37:00::","synset_id":"anticipate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"look_for%2:35:00::","call_ids":["senseval2.d001.s070.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"look_for%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.003405,"input_uncached_usd":0.001935,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7024713749997318}
{"item_id":"senseval2.d001.s070.t009","gold_sense_keys":["common%5:00:00:shared:00"],"candidates":[{"index":1,"sense_key":"common%3:00:01::","synset_id":"common.a.02"},{"index":2,"sense_key":"common%5:00:00:inferior:02","synset_id":"coarse.s.02"},{"index":3,"sense_key":"common%3:00:02::","synset_id":"common.a.01"},{"index":4,"sense_key":"common%5:00:00:ordinary:00","synset_id":"common.s.07"},{"index":5,"sense_key":"common%5:00:00:shared:00","synset_id":"common.s.01"},{"index":6,"sense_key":"common%5:00:00:unrefined:01","synset_id":"coarse.s.01"},{"index":7,"sense_key":"common%5:00:00:lowborn:00","synset_id":"common.s.04"},{"index":8,"sense_key":"common%5:00:00:familiar:02","synset_id":"common.s.02"},{"index":9,"sense_key":"common%5:00:00:informal:02","synset_id":"common.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"common%5:00:00:shared:00","call_ids":["senseval2.d001.s070.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"common%5:00:00:shared:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.006175,"input_uncached_usd":0.0036850000000000003,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7412148750154302}
{"item_id":"senseval2.d001.s070.t010","gold_sense_keys":["bit%1:17:00::","bit%1:23:01::"],"candidates":[{"index":1,"sense_key":"bit%1:11:00::","synset_id":"piece.n.05"},{"index":2,"sense_key":"bit%1:06:01::","synset_id":"bit.n.05"},{"index":3,"sense_key":"bit%1:10:01::","synset_id":"snatch.n.01"},{"index":4,"sense_key":"bit%1:13:00::","synset_id":"morsel.n.02"},{"index":5,"sense_key":"bit%1:06:02::","synset_id":"bit.n.10"},{"index":6,"sense_key":"bit%1:23:01::","synset_id":"spot.n.10"},{"index":7,"sense_key":"bit%1:23:00::","synset_id":"bit.n.06"},{"index":8,"sense_key":"bit%1:17:00::","synset_id":"bit.n.02"},{"index":9,"sense_key":"bit%1:28:00::","synset_id":"moment.n.02"},{"index":10,"sense_key":"bit%1:10:00::","synset_id":"act.n.04"},{"index":11,"sense_key":"bit%1:06:00::","synset_id":"bit.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"bit%1:23:01::","call_ids":["senseval2.d001.s070.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"bit%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":784,"cached_input_tokens":0,"output_tokens":353,"reasoning_output_tokens":337},"cost":{"currency":"USD","total_usd":0.014510000000000002,"input_uncached_usd":0.003920000000000001,"input_cached_usd":0.0,"output_usd":0.01059,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.004271832993254}
{"item_id":"senseval2.d001.s070.t011","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":2,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":3,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"},{"index":4,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s070.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":120},"cost":{"currency":"USD","total_usd":0.006455000000000001,"input_uncached_usd":0.0023750000000000004,"input_cached_usd":0.0,"output_usd":0.00408,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9354065409861505}
{"item_id":"senseval2.d001.s070.t012","gold_sense_keys":["material%1:27:00::"],"candidates":[{"index":1,"sense_key":"material%1:10:00::","synset_id":"material.n.02"},{"index":2,"sense_key":"material%1:27:00::","synset_id":"material.n.01"},{"index":3,"sense_key":"material%1:06:00::","synset_id":"fabric.n.01"},{"index":4,"sense_key":"material%1:18:00::","synset_id":"material.n.05"},{"index":5,"sense_key":"material%1:06:01::","synset_id":"material.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"material%1:27:00::","call_ids":["senseval2.d001.s070.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"material%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.0052899999999999996,"input_uncached_usd":0.00268,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8757769590010867}
{"item_id":"senseval2.d001.s070.t013","gold_sense_keys":["lose%2:40:02::"],"candidates":[{"index":1,"sense_key":"lose%2:40:02::","synset_id":"lose.v.01"},{"index":2,"sense_key":"lose%2:39:00::","synset_id":"lose.v.06"},{"index":3,"sense_key":"lose%2:35:00::","synset_id":"misplace.v.01"},{"index":4,"sense_key":"lose%2:40:01::","synset_id":"lose.v.05"},{"index":5,"sense_key":"lose%2:37:00::","synset_id":"lose.v.03"},{"index":6,"sense_key":"lose%2:39:01::","synset_id":"miss.v.01"},{"index":7,"sense_key":"lose%2:40:06::","synset_id":"lose.v.07"},{"index":8,"sense_key":"lose%2:33:00::","synset_id":"lose.v.02"},{"index":9,"sense_key":"lose%2:40:00::","synset_id":"lose.v.08"},{"index":10,"sense_key":"lose%2:30:03::","synset_id":"suffer.v.11"},{"index":11,"sense_key":"lose%2:33:01::","synset_id":"fall_back.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lose%2:40:02::","call_ids":["senseval2.d001.s070.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lose%2:40:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.007285,"input_uncached_usd":0.0038050000000000002,"input_cached_usd":0.0,"output_usd":0.00348,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5505104169715196}
{"item_id":"senseval2.d001.s070.t015","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":2,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":5,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":6,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":7,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s070.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.00425,"input_uncached_usd":0.00284,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.100168374949135}
{"item_id":"senseval2.d001.s071.t000","gold_sense_keys":["piece%1:06:00::","piece%1:17:00::"],"candidates":[{"index":1,"sense_key":"piece%1:11:00::","synset_id":"piece.n.05"},{"index":2,"sense_key":"piece%1:21:00::","synset_id":"slice.n.01"},{"index":3,"sense_key":"piece%1:06:02::","synset_id":"man.n.10"},{"index":4,"sense_key":"piece%1:06:04::","synset_id":"objet_d'art.n.01"},{"index":5,"sense_key":"piece%1:10:02::","synset_id":"piece.n.06"},{"index":6,"sense_key":"piece%1:28:00::","synset_id":"while.n.01"},{"index":7,"sense_key":"piece%1:17:00::","synset_id":"part.n.03"},{"index":8,"sense_key":"piece%1:06:00::","synset_id":"piece.n.01"},{"index":9,"sense_key":"piece%1:13:00::","synset_id":"piece.n.08"},{"index":10,"sense_key":"piece%1:10:01::","synset_id":"musical_composition.n.01"},{"index":11,"sense_key":"piece%1:06:03::","synset_id":"firearm.n.01"},{"index":12,"sense_key":"piece%1:07:00::","synset_id":"piece.n.09"},{"index":13,"sense_key":"piece%1:06:05::","synset_id":"piece.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"piece%1:17:00::","call_ids":["senseval2.d001.s071.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"piece%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":857,"cached_input_tokens":0,"output_tokens":220,"reasoning_output_tokens":204},"cost":{"currency":"USD","total_usd":0.010885000000000002,"input_uncached_usd":0.004285000000000001,"input_cached_usd":0.0,"output_usd":0.0066,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.531665707938373}
{"item_id":"senseval2.d001.s071.t002","gold_sense_keys":["probably%4:02:00::"],"candidates":[{"index":1,"sense_key":"probably%4:02:01::","synset_id":"credibly.r.01"},{"index":2,"sense_key":"probably%4:02:00::","synset_id":"probably.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"probably%4:02:00::","call_ids":["senseval2.d001.s071.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"probably%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":225,"reasoning_output_tokens":209},"cost":{"currency":"USD","total_usd":0.008795,"input_uncached_usd":0.0020450000000000004,"input_cached_usd":0.0,"output_usd":0.00675,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6537962920265272}
{"item_id":"senseval2.d001.s071.t003","gold_sense_keys":["constitute%2:42:00::"],"candidates":[{"index":1,"sense_key":"constitute%2:36:00::","synset_id":"establish.v.02"},{"index":2,"sense_key":"constitute%2:42:00::","synset_id":"constitute.v.01"},{"index":3,"sense_key":"constitute%2:42:03::","synset_id":"form.v.02"},{"index":4,"sense_key":"constitute%2:41:00::","synset_id":"appoint.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"constitute%2:42:00::","call_ids":["senseval2.d001.s071.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"constitute%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":2057,"reasoning_output_tokens":2023},"cost":{"currency":"USD","total_usd":0.064095,"input_uncached_usd":0.0023850000000000004,"input_cached_usd":0.0,"output_usd":0.06171,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":35.088624791940674}
{"item_id":"senseval2.d001.s072.t000","gold_sense_keys":["find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":2,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":3,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":4,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":5,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":6,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":7,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":8,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":9,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":12,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":13,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":14,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":15,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"find%2:36:00::","call_ids":["senseval2.d001.s072.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"find%2:36:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1120,"cached_input_tokens":0,"output_tokens":9851,"reasoning_output_tokens":9730},"cost":{"currency":"USD","total_usd":0.30113,"input_uncached_usd":0.005600000000000001,"input_cached_usd":0.0,"output_usd":0.29553,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":167.11359420791268}
{"item_id":"senseval2.d001.s072.t001","gold_sense_keys":["last%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"last%5:00:00:unalterable:00","synset_id":"final.s.03"},{"index":2,"sense_key":"last%3:00:00::","synset_id":"last.a.02"},{"index":3,"sense_key":"last%5:00:00:dying:00","synset_id":"last.s.04"},{"index":4,"sense_key":"last%5:00:00:ultimate:00","synset_id":"final.s.02"},{"index":5,"sense_key":"last%5:00:00:past:00","synset_id":"last.s.01"},{"index":6,"sense_key":"last%5:00:00:high:02","synset_id":"last.s.06"},{"index":7,"sense_key":"last%5:00:00:worst:00","synset_id":"last.s.08"},{"index":8,"sense_key":"last%5:00:00:closing:00","synset_id":"concluding.s.01"},{"index":9,"sense_key":"last%5:00:00:unlikely:00","synset_id":"last.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"last%5:00:00:past:00","call_ids":["senseval2.d001.s072.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"last%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":741,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.006045,"input_uncached_usd":0.0037050000000000004,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.972125209053047}
{"item_id":"senseval2.d001.s072.t003","gold_sense_keys":["dubious%5:00:00:uncertain:02","dubious%5:00:00:unconvinced:00"],"candidates":[{"index":1,"sense_key":"dubious%5:00:00:uncertain:02","synset_id":"doubtful.s.02"},{"index":2,"sense_key":"dubious%5:00:00:unconvinced:00","synset_id":"dubious.s.03"},{"index":3,"sense_key":"dubious%5:00:00:questionable:00","synset_id":"doubtful.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dubious%5:00:00:unconvinced:00","call_ids":["senseval2.d001.s072.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dubious%5:00:00:unconvinced:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.004685,"input_uncached_usd":0.002405,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9567607080098242}
{"item_id":"senseval2.d001.s072.t004","gold_sense_keys":["search%1:04:00::","search%1:04:02::"],"candidates":[{"index":1,"sense_key":"search%1:04:01::","synset_id":"search.n.05"},{"index":2,"sense_key":"search%1:22:00::","synset_id":"search.n.03"},{"index":3,"sense_key":"search%1:09:00::","synset_id":"search.n.04"},{"index":4,"sense_key":"search%1:04:00::","synset_id":"search.n.01"},{"index":5,"sense_key":"search%1:04:02::","synset_id":"search.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"search%1:04:00::","call_ids":["senseval2.d001.s072.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"search%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":499,"reasoning_output_tokens":483},"cost":{"currency":"USD","total_usd":0.017570000000000002,"input_uncached_usd":0.0026000000000000003,"input_cached_usd":0.0,"output_usd":0.01497,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.148105291998945}
{"item_id":"senseval2.d001.s073.t000","gold_sense_keys":["doubt%1:07:00::"],"candidates":[{"index":1,"sense_key":"doubt%1:07:00::","synset_id":"doubt.n.02"},{"index":2,"sense_key":"doubt%1:09:00::","synset_id":"doubt.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"doubt%1:07:00::","call_ids":["senseval2.d001.s073.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"doubt%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01791,"input_uncached_usd":0.0020700000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.020187375019304}
{"item_id":"senseval2.d001.s073.t001","gold_sense_keys":["stem%2:42:00::"],"candidates":[{"index":1,"sense_key":"stem%2:30:01::","synset_id":"stem.v.04"},{"index":2,"sense_key":"stem%2:30:00::","synset_id":"stem.v.03"},{"index":3,"sense_key":"stem%2:42:00::","synset_id":"stem.v.01"},{"index":4,"sense_key":"stem%2:42:03::","synset_id":"stem.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"stem%2:42:00::","call_ids":["senseval2.d001.s073.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"stem%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.003975,"input_uncached_usd":0.002295,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5891502500744537}
{"item_id":"senseval2.d001.s073.t002","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":2,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":3,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"},{"index":4,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fact%1:09:01::","call_ids":["senseval2.d001.s073.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fact%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":2253,"reasoning_output_tokens":2213},"cost":{"currency":"USD","total_usd":0.07000500000000001,"input_uncached_usd":0.002415,"input_cached_usd":0.0,"output_usd":0.06759,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":41.392468291916884}
{"item_id":"senseval2.d001.s073.t003","gold_sense_keys":["several%5:00:00:some:00"],"candidates":[{"index":1,"sense_key":"several%5:00:00:different:00","synset_id":"several.s.03"},{"index":2,"sense_key":"several%5:00:00:some:00","synset_id":"several.s.01"},{"index":3,"sense_key":"several%5:00:00:individual:00","synset_id":"respective.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"several%5:00:00:some:00","call_ids":["senseval2.d001.s073.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"several%5:00:00:some:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.0040950000000000005,"input_uncached_usd":0.0022050000000000004,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6872767080785707}
{"item_id":"senseval2.d001.s073.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":3,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d001.s073.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004535000000000001,"input_uncached_usd":0.0025250000000000003,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3036050000227988}
{"item_id":"senseval2.d001.s073.t005","gold_sense_keys":["earlier%4:02:00::"],"candidates":[{"index":1,"sense_key":"earlier%4:02:00::","synset_id":"earlier.r.01"},{"index":2,"sense_key":"earlier%4:02:02::","synset_id":"sooner.r.01"},{"index":3,"sense_key":"earlier%4:02:04::","synset_id":"in_the_first_place.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"earlier%4:02:00::","call_ids":["senseval2.d001.s073.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"earlier%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.005695,"input_uncached_usd":0.002245,"input_cached_usd":0.0,"output_usd":0.00345,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.032197249936871}
{"item_id":"senseval2.d001.s073.t007","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":2,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":3,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":4,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"},{"index":5,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":6,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":7,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":8,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":9,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":10,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":13,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":14,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":15,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":16,"sense_key":"find%2:40:00::","synset_id":"find.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"find%2:32:00::","call_ids":["senseval2.d001.s073.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"find%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1101,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.036885,"input_uncached_usd":0.005505,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":15.1803985000588}
{"item_id":"senseval2.d001.s073.t008","gold_sense_keys":["experiment%1:04:00::"],"candidates":[{"index":1,"sense_key":"experiment%1:04:00::","synset_id":"experiment.n.01"},{"index":2,"sense_key":"experiment%1:04:01::","synset_id":"experiment.n.03"},{"index":3,"sense_key":"experiment%1:09:00::","synset_id":"experiment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"experiment%1:04:00::","call_ids":["senseval2.d001.s073.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"experiment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.00474,"input_uncached_usd":0.0021000000000000003,"input_cached_usd":0.0,"output_usd":0.00264,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.009148749988526}
{"item_id":"senseval2.d001.s073.t009","gold_sense_keys":["mouse%1:05:00::"],"candidates":[{"index":1,"sense_key":"mouse%1:18:00::","synset_id":"mouse.n.03"},{"index":2,"sense_key":"mouse%1:06:00::","synset_id":"mouse.n.04"},{"index":3,"sense_key":"mouse%1:05:00::","synset_id":"mouse.n.01"},{"index":4,"sense_key":"mouse%1:26:00::","synset_id":"shiner.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"mouse%1:05:00::","call_ids":["senseval2.d001.s073.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"mouse%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.0039000000000000003,"input_uncached_usd":0.0024300000000000003,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7995418750215322}
{"item_id":"senseval2.d001.s073.t011","gold_sense_keys":["call%2:32:02::"],"candidates":[{"index":1,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":2,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":3,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":4,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":5,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":6,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":7,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":8,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":9,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":10,"sense_key":"call%2:29:05::","synset_id":"call.v.28"},{"index":11,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":12,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":13,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":14,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":15,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":16,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":17,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":18,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":19,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":20,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":21,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":22,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":23,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":24,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":25,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":26,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":27,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":28,"sense_key":"call%2:41:12::","synset_id":"call.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"call%2:32:02::","call_ids":["senseval2.d001.s073.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"call%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1487,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.009265,"input_uncached_usd":0.007435000000000001,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":76.99768591695465}
{"item_id":"senseval2.d001.s073.t012","gold_sense_keys":["transform%2:30:00::"],"candidates":[{"index":1,"sense_key":"transform%2:30:03::","synset_id":"transform.v.03"},{"index":2,"sense_key":"transform%2:30:04::","synset_id":"transform.v.07"},{"index":3,"sense_key":"transform%2:30:05::","synset_id":"transform.v.06"},{"index":4,"sense_key":"transform%2:30:02::","synset_id":"transform.v.01"},{"index":5,"sense_key":"transform%2:30:01::","synset_id":"translate.v.02"},{"index":6,"sense_key":"transform%2:30:00::","synset_id":"transform.v.02"},{"index":7,"sense_key":"transform%2:30:06::","synset_id":"transform.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"transform%2:30:00::","call_ids":["senseval2.d001.s073.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"transform%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":576,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":133},"cost":{"currency":"USD","total_usd":0.007350000000000001,"input_uncached_usd":0.00288,"input_cached_usd":0.0,"output_usd":0.00447,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.251799707999453}
{"item_id":"senseval2.d001.s073.t013","gold_sense_keys":["normal%3:00:01::"],"candidates":[{"index":1,"sense_key":"normal%3:00:01::","synset_id":"normal.a.01"},{"index":2,"sense_key":"normal%3:00:02::","synset_id":"normal.a.02"},{"index":3,"sense_key":"normal%5:00:00:perpendicular:00","synset_id":"normal.s.01"},{"index":4,"sense_key":"normal%3:00:03::","synset_id":"normal.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"normal%3:00:01::","call_ids":["senseval2.d001.s073.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"normal%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":117},"cost":{"currency":"USD","total_usd":0.00619,"input_uncached_usd":0.00223,"input_cached_usd":0.0,"output_usd":0.00396,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.997707291971892}
{"item_id":"senseval2.d001.s073.t014","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":3,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":4,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":5,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s073.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":28},"cost":{"currency":"USD","total_usd":0.00423,"input_uncached_usd":0.0029100000000000003,"input_cached_usd":0.0,"output_usd":0.00132,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7433960000053048}
{"item_id":"senseval2.d001.s073.t015","gold_sense_keys":["cancerous%5:00:00:malignant:02"],"candidates":[{"index":1,"sense_key":"cancerous%5:00:00:malignant:02","synset_id":"cancerous.s.01"},{"index":2,"sense_key":"cancerous%5:00:00:malign:00","synset_id":"cancerous.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancerous%5:00:00:malignant:02","call_ids":["senseval2.d001.s073.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancerous%5:00:00:malignant:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":401,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.0038050000000000002,"input_uncached_usd":0.0020050000000000003,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.311188374995254}
{"item_id":"senseval2.d001.s073.t016","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval2.d001.s073.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.005005000000000001,"input_uncached_usd":0.0020350000000000004,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1566510419361293}
{"item_id":"senseval2.d001.s074.t000","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":2,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":3,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":4,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":5,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":6,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":7,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":8,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":9,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":10,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":11,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":12,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":13,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"},{"index":16,"sense_key":"find%2:39:05::","synset_id":"find.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"find%2:39:02::","call_ids":["senseval2.d001.s074.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"find%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1101,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.036885,"input_uncached_usd":0.005505,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":16.27394525008276}
{"item_id":"senseval2.d001.s074.t001","gold_sense_keys":["exactly%4:02:01::"],"candidates":[{"index":1,"sense_key":"exactly%4:02:01::","synset_id":"precisely.r.01"},{"index":2,"sense_key":"exactly%4:02:02::","synset_id":"precisely.r.02"},{"index":3,"sense_key":"exactly%4:02:04::","synset_id":"precisely.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"exactly%4:02:01::","call_ids":["senseval2.d001.s074.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"exactly%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.0043300000000000005,"input_uncached_usd":0.00232,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.800296708010137}
{"item_id":"senseval2.d001.s074.t002","gold_sense_keys":["same%3:00:00::"],"candidates":[{"index":1,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"same%3:00:02::","call_ids":["senseval2.d001.s074.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"same%3:00:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.005265000000000001,"input_uncached_usd":0.0023550000000000003,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0074319589184597}
{"item_id":"senseval2.d001.s074.t003","gold_sense_keys":["spot%1:15:01::"],"candidates":[{"index":1,"sense_key":"spot%1:06:01::","synset_id":"spotlight.n.02"},{"index":2,"sense_key":"spot%1:10:00::","synset_id":"spot.n.06"},{"index":3,"sense_key":"spot%1:23:00::","synset_id":"spot.n.10"},{"index":4,"sense_key":"spot%1:26:00::","synset_id":"touch.n.08"},{"index":5,"sense_key":"spot%1:04:00::","synset_id":"blot.n.02"},{"index":6,"sense_key":"spot%1:06:00::","synset_id":"spot.n.07"},{"index":7,"sense_key":"spot%1:07:00::","synset_id":"smudge.n.02"},{"index":8,"sense_key":"spot%1:07:02::","synset_id":"point.n.14"},{"index":9,"sense_key":"spot%1:10:01::","synset_id":"spot.n.02"},{"index":10,"sense_key":"spot%1:04:01::","synset_id":"position.n.06"},{"index":11,"sense_key":"spot%1:06:02::","synset_id":"spot.n.13"},{"index":12,"sense_key":"spot%1:10:02::","synset_id":"spot.n.11"},{"index":13,"sense_key":"spot%1:07:01::","synset_id":"spot.n.05"},{"index":14,"sense_key":"spot%1:15:01::","synset_id":"topographic_point.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"spot%1:15:01::","call_ids":["senseval2.d001.s074.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"spot%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":898,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.00767,"input_uncached_usd":0.00449,"input_cached_usd":0.0,"output_usd":0.00318,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4784472920000553}
{"item_id":"senseval2.d001.s075.t000","gold_sense_keys":["say%2:32:01::"],"candidates":[{"index":1,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":2,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":5,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":6,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":7,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":8,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":9,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":10,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":11,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d001.s075.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0201,"input_uncached_usd":0.004260000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.737514791893773}
{"item_id":"senseval2.d001.s075.t002","gold_sense_keys":["cause%2:36:00::"],"candidates":[{"index":1,"sense_key":"cause%2:36:00::","synset_id":"cause.v.01"},{"index":2,"sense_key":"cause%2:32:00::","synset_id":"induce.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cause%2:36:00::","call_ids":["senseval2.d001.s075.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cause%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.00489,"input_uncached_usd":0.0024000000000000002,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.844611374894157}
{"item_id":"senseval2.d001.s075.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":2,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":3,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":4,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":5,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s075.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.00471,"input_uncached_usd":0.00285,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9108384170103818}
{"item_id":"senseval2.d001.s075.t004","gold_sense_keys":["promote%2:41:01::"],"candidates":[{"index":1,"sense_key":"promote%2:32:00::","synset_id":"advertise.v.02"},{"index":2,"sense_key":"promote%2:41:01::","synset_id":"promote.v.01"},{"index":3,"sense_key":"promote%2:41:00::","synset_id":"promote.v.02"},{"index":4,"sense_key":"promote%2:33:01::","synset_id":"promote.v.04"},{"index":5,"sense_key":"promote%2:33:00::","synset_id":"promote.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"promote%2:41:01::","call_ids":["senseval2.d001.s075.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"promote%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00495,"input_uncached_usd":0.0031200000000000004,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8309782500145957}
{"item_id":"senseval2.d001.s075.t005","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":2,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":3,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":4,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"},{"index":5,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":6,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":7,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"growth%1:22:02::","call_ids":["senseval2.d001.s075.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"growth%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019145,"input_uncached_usd":0.003305,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.062796541024}
{"item_id":"senseval2.d001.s075.t007","gold_sense_keys":["look_for%2:35:00::"],"candidates":[{"index":1,"sense_key":"look_for%2:35:00::","synset_id":"search.v.01"},{"index":2,"sense_key":"look_for%2:37:00::","synset_id":"anticipate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"look_for%2:35:00::","call_ids":["senseval2.d001.s075.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"look_for%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.003980000000000001,"input_uncached_usd":0.0023000000000000004,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9613783749518916}
{"item_id":"senseval2.d001.s075.t009","gold_sense_keys":["suppress%2:30:12::"],"candidates":[{"index":1,"sense_key":"suppress%2:30:00::","synset_id":"suppress.v.01"},{"index":2,"sense_key":"suppress%2:30:12::","synset_id":"suppress.v.05"},{"index":3,"sense_key":"suppress%2:41:01::","synset_id":"oppress.v.01"},{"index":4,"sense_key":"suppress%2:41:00::","synset_id":"inhibit.v.04"},{"index":5,"sense_key":"suppress%2:31:00::","synset_id":"suppress.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"suppress%2:30:12::","call_ids":["senseval2.d001.s075.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"suppress%2:30:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.00601,"input_uncached_usd":0.00286,"input_cached_usd":0.0,"output_usd":0.00315,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5560077079571784}
{"item_id":"senseval2.d001.s075.t010","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":2,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":3,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":4,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":5,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":6,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":7,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"growth%1:22:02::","call_ids":["senseval2.d001.s075.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"growth%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":377,"reasoning_output_tokens":362},"cost":{"currency":"USD","total_usd":0.014615000000000001,"input_uncached_usd":0.003305,"input_cached_usd":0.0,"output_usd":0.01131,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.006383458035998}
{"item_id":"senseval2.d001.s076.t001","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":2,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":3,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"},{"index":4,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"compare%2:31:00::","call_ids":["senseval2.d001.s076.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.0047550000000000005,"input_uncached_usd":0.0027150000000000004,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.972075124969706}
{"item_id":"senseval2.d001.s076.t003","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"},{"index":2,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":3,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":4,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":5,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":6,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":7,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":8,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":11,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":12,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":13,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":14,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":15,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":16,"sense_key":"find%2:31:10::","synset_id":"find.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"find%2:39:02::","call_ids":["senseval2.d001.s076.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"find%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1165,"cached_input_tokens":0,"output_tokens":4634,"reasoning_output_tokens":4573},"cost":{"currency":"USD","total_usd":0.144845,"input_uncached_usd":0.005825,"input_cached_usd":0.0,"output_usd":0.13902,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":85.29118512501009}
{"item_id":"senseval2.d001.s076.t004","gold_sense_keys":["human%3:01:01::"],"candidates":[{"index":1,"sense_key":"human%3:01:00::","synset_id":"human.a.01"},{"index":2,"sense_key":"human%3:01:01::","synset_id":"human.a.02"},{"index":3,"sense_key":"human%3:00:00::","synset_id":"human.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"human%3:01:01::","call_ids":["senseval2.d001.s076.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"human%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":203,"reasoning_output_tokens":187},"cost":{"currency":"USD","total_usd":0.008475000000000002,"input_uncached_usd":0.0023850000000000004,"input_cached_usd":0.0,"output_usd":0.00609,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8600582500221208}
{"item_id":"senseval2.d001.s076.t008","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":2,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":3,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":4,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":5,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":6,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":7,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":8,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":9,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":10,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"},{"index":11,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":12,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":13,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":14,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":15,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":16,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"find%2:32:00::","call_ids":["senseval2.d001.s076.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"find%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1165,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.021665,"input_uncached_usd":0.005825,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.975462042028084}
{"item_id":"senseval2.d001.s076.t009","gold_sense_keys":["identical%5:00:00:same:00"],"candidates":[{"index":1,"sense_key":"identical%5:00:00:isotropic:00","synset_id":"identical.s.03"},{"index":2,"sense_key":"identical%5:00:00:same:02","synset_id":"identical.s.02"},{"index":3,"sense_key":"identical%5:00:00:same:00","synset_id":"identical.s.01"},{"index":4,"sense_key":"identical%3:00:00::","synset_id":"identical.a.03"},{"index":5,"sense_key":"identical%5:00:00:congruent:00","synset_id":"identical.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"identical%5:00:00:same:00","call_ids":["senseval2.d001.s076.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"identical%5:00:00:same:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":420,"reasoning_output_tokens":405},"cost":{"currency":"USD","total_usd":0.015605000000000003,"input_uncached_usd":0.0030050000000000003,"input_cached_usd":0.0,"output_usd":0.0126,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.593589041964151}
{"item_id":"senseval2.d001.s076.t010","gold_sense_keys":["turn_out%2:42:00::","turn_out%2:42:01::"],"candidates":[{"index":1,"sense_key":"turn_out%2:29:00::","synset_id":"get_up.v.02"},{"index":2,"sense_key":"turn_out%2:30:00::","synset_id":"turn_out.v.05"},{"index":3,"sense_key":"turn_out%2:35:00::","synset_id":"eject.v.01"},{"index":4,"sense_key":"turn_out%2:41:03::","synset_id":"turn_out.v.08"},{"index":5,"sense_key":"turn_out%2:42:02::","synset_id":"turn_out.v.04"},{"index":6,"sense_key":"turn_out%2:36:00::","synset_id":"bear.v.05"},{"index":7,"sense_key":"turn_out%2:38:00::","synset_id":"turn_out.v.10"},{"index":8,"sense_key":"turn_out%2:35:01::","synset_id":"switch_off.v.01"},{"index":9,"sense_key":"turn_out%2:36:01::","synset_id":"turn_out.v.03"},{"index":10,"sense_key":"turn_out%2:42:01::","synset_id":"turn_out.v.02"},{"index":11,"sense_key":"turn_out%2:42:00::","synset_id":"prove.v.01"},{"index":12,"sense_key":"turn_out%2:40:03::","synset_id":"turn_out.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"turn_out%2:42:01::","call_ids":["senseval2.d001.s076.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"turn_out%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":895,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":123},"cost":{"currency":"USD","total_usd":0.008645,"input_uncached_usd":0.004475000000000001,"input_cached_usd":0.0,"output_usd":0.00417,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.606767166987993}
{"item_id":"senseval2.d001.s076.t011","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":2,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s076.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.00461,"input_uncached_usd":0.00287,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8447851249948144}
{"item_id":"senseval2.d001.s076.t012","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":2,"sense_key":"study%1:10:01::","synset_id":"study.n.10"},{"index":3,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":4,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":5,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":6,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":7,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":8,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":9,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":10,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"study%1:04:00::","call_ids":["senseval2.d001.s076.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":787,"cached_input_tokens":0,"output_tokens":252,"reasoning_output_tokens":236},"cost":{"currency":"USD","total_usd":0.011495000000000002,"input_uncached_usd":0.003935,"input_cached_usd":0.0,"output_usd":0.00756,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.35136699990835}
{"item_id":"senseval2.d001.s076.t014","gold_sense_keys":["observe%2:39:00::"],"candidates":[{"index":1,"sense_key":"observe%2:32:00::","synset_id":"note.v.01"},{"index":2,"sense_key":"observe%2:41:00::","synset_id":"watch.v.02"},{"index":3,"sense_key":"observe%2:41:04::","synset_id":"observe.v.09"},{"index":4,"sense_key":"observe%2:39:00::","synset_id":"observe.v.04"},{"index":5,"sense_key":"observe%2:31:00::","synset_id":"observe.v.08"},{"index":6,"sense_key":"observe%2:41:02::","synset_id":"observe.v.06"},{"index":7,"sense_key":"observe%2:39:01::","synset_id":"note.v.03"},{"index":8,"sense_key":"observe%2:39:02::","synset_id":"detect.v.01"},{"index":9,"sense_key":"observe%2:41:01::","synset_id":"respect.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"observe%2:39:00::","call_ids":["senseval2.d001.s076.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"observe%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":777,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019725,"input_uncached_usd":0.0038850000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.400061625055969}
{"item_id":"senseval2.d001.s076.t015","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s076.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.004525,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0088927920442075}
{"item_id":"senseval2.d001.s076.t016","gold_sense_keys":["form%1:07:03::"],"candidates":[{"index":1,"sense_key":"form%1:14:00::","synset_id":"class.n.02"},{"index":2,"sense_key":"form%1:10:00::","synset_id":"form.n.01"},{"index":3,"sense_key":"form%1:06:01::","synset_id":"form.n.16"},{"index":4,"sense_key":"form%1:07:02::","synset_id":"form.n.14"},{"index":5,"sense_key":"form%1:08:00::","synset_id":"human_body.n.01"},{"index":6,"sense_key":"form%1:07:00::","synset_id":"form.n.07"},{"index":7,"sense_key":"form%1:26:00::","synset_id":"phase.n.02"},{"index":8,"sense_key":"form%1:07:03::","synset_id":"form.n.11"},{"index":9,"sense_key":"form%1:03:00::","synset_id":"shape.n.02"},{"index":10,"sense_key":"form%1:06:00::","synset_id":"mannequin.n.02"},{"index":11,"sense_key":"form%1:10:01::","synset_id":"form.n.08"},{"index":12,"sense_key":"form%1:07:01::","synset_id":"shape.n.01"},{"index":13,"sense_key":"form%1:09:00::","synset_id":"form.n.03"},{"index":14,"sense_key":"form%1:09:01::","synset_id":"kind.n.01"},{"index":15,"sense_key":"form%1:14:02::","synset_id":"form.n.09"},{"index":16,"sense_key":"form%1:10:02::","synset_id":"form.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"form%1:09:01::","call_ids":["senseval2.d001.s076.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"form%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1106,"cached_input_tokens":0,"output_tokens":253,"reasoning_output_tokens":237},"cost":{"currency":"USD","total_usd":0.01312,"input_uncached_usd":0.00553,"input_cached_usd":0.0,"output_usd":0.00759,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.959969624993391}
{"item_id":"senseval2.d001.s077.t000","gold_sense_keys":["discovery%1:04:00::","discovery%1:09:00::","discovery%1:10:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"},{"index":4,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discovery%1:09:00::","call_ids":["senseval2.d001.s077.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discovery%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018395,"input_uncached_usd":0.0025550000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.295057374984026}
{"item_id":"senseval2.d001.s077.t001","gold_sense_keys":["suddenly%4:02:00::","suddenly%4:02:02::"],"candidates":[{"index":1,"sense_key":"suddenly%4:02:02::","synset_id":"suddenly.r.01"},{"index":2,"sense_key":"suddenly%4:02:00::","synset_id":"abruptly.r.01"},{"index":3,"sense_key":"suddenly%4:02:04::","synset_id":"on_the_spur_of_the_moment.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"suddenly%4:02:02::","call_ids":["senseval2.d001.s077.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"suddenly%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":445,"reasoning_output_tokens":429},"cost":{"currency":"USD","total_usd":0.015945,"input_uncached_usd":0.002595,"input_cached_usd":0.0,"output_usd":0.01335,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":15.782466833945364}
{"item_id":"senseval2.d001.s077.t002","gold_sense_keys":["put%2:35:00::"],"candidates":[{"index":1,"sense_key":"put%2:35:05::","synset_id":"put.v.02"},{"index":2,"sense_key":"put%2:40:00::","synset_id":"invest.v.01"},{"index":3,"sense_key":"put%2:35:00::","synset_id":"put.v.01"},{"index":4,"sense_key":"put%2:32:00::","synset_id":"frame.v.04"},{"index":5,"sense_key":"put%2:31:00::","synset_id":"arrange.v.07"},{"index":6,"sense_key":"put%2:31:13::","synset_id":"place.v.11"},{"index":7,"sense_key":"put%2:34:00::","synset_id":"put.v.04"},{"index":8,"sense_key":"put%2:39:13::","synset_id":"put.v.07"},{"index":9,"sense_key":"put%2:36:13::","synset_id":"put.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"put%2:35:00::","call_ids":["senseval2.d001.s077.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"put%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":755,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.007285,"input_uncached_usd":0.003775,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.461642457987182}
{"item_id":"senseval2.d001.s077.t005","gold_sense_keys":["right%4:02:04::","right%4:02:09::"],"candidates":[{"index":1,"sense_key":"right%4:02:01::","synset_id":"properly.r.01"},{"index":2,"sense_key":"right%4:02:03::","synset_id":"right.r.04"},{"index":3,"sense_key":"right%4:02:06::","synset_id":"mighty.r.01"},{"index":4,"sense_key":"right%4:02:08::","synset_id":"right.r.02"},{"index":5,"sense_key":"right%4:02:07::","synset_id":"right.r.07"},{"index":6,"sense_key":"right%4:02:05::","synset_id":"justly.r.02"},{"index":7,"sense_key":"right%4:02:09::","synset_id":"right.r.03"},{"index":8,"sense_key":"right%4:02:00::","synset_id":"right.r.06"},{"index":9,"sense_key":"right%4:02:04::","synset_id":"right.r.01"},{"index":10,"sense_key":"right%4:02:02::","synset_id":"correctly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"right%4:02:04::","call_ids":["senseval2.d001.s077.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"right%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":756,"cached_input_tokens":0,"output_tokens":3117,"reasoning_output_tokens":3072},"cost":{"currency":"USD","total_usd":0.09729000000000002,"input_uncached_usd":0.0037800000000000004,"input_cached_usd":0.0,"output_usd":0.09351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":60.415631582960486}
{"item_id":"senseval2.d001.s077.t006","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":2,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":3,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":4,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":5,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s077.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.00429,"input_uncached_usd":0.0029100000000000003,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7847900000633672}
{"item_id":"senseval2.d001.s077.t007","gold_sense_keys":["formation%1:22:00::"],"candidates":[{"index":1,"sense_key":"formation%1:04:01::","synset_id":"constitution.n.02"},{"index":2,"sense_key":"formation%1:14:00::","synset_id":"formation.n.01"},{"index":3,"sense_key":"formation%1:04:00::","synset_id":"formation.n.02"},{"index":4,"sense_key":"formation%1:17:00::","synset_id":"geological_formation.n.01"},{"index":5,"sense_key":"formation%1:22:00::","synset_id":"formation.n.06"},{"index":6,"sense_key":"formation%1:06:00::","synset_id":"formation.n.05"},{"index":7,"sense_key":"formation%1:04:02::","synset_id":"formation.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"formation%1:22:00::","call_ids":["senseval2.d001.s077.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"formation%1:22:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.00544,"input_uncached_usd":0.0031300000000000004,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7813779170392081}
{"item_id":"senseval2.d001.s077.t008","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":2,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":3,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":4,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":5,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":6,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":7,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":8,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":9,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":10,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":11,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d001.s077.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":862,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02015,"input_uncached_usd":0.0043100000000000005,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.887225041980855}
{"item_id":"senseval2.d001.s077.t009","gold_sense_keys":["leader%1:18:00::"],"candidates":[{"index":1,"sense_key":"leader%1:18:00::","synset_id":"leader.n.01"},{"index":2,"sense_key":"leader%1:06:00::","synset_id":"drawing_card.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"leader%1:18:00::","call_ids":["senseval2.d001.s077.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"leader%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00402,"input_uncached_usd":0.00222,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5738295830087736}
{"item_id":"senseval2.d001.s078.t000","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"},{"index":2,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":3,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval2.d001.s078.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":205,"reasoning_output_tokens":189},"cost":{"currency":"USD","total_usd":0.008740000000000001,"input_uncached_usd":0.0025900000000000003,"input_cached_usd":0.0,"output_usd":0.00615,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.677753582946025}
{"item_id":"senseval2.d001.s078.t001","gold_sense_keys":["now%4:02:00::","now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":2,"sense_key":"now%4:02:06::","synset_id":"now.r.07"},{"index":3,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":4,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":5,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":6,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":7,"sense_key":"now%4:02:02::","synset_id":"now.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"now%4:02:04::","call_ids":["senseval2.d001.s078.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"now%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":706,"cached_input_tokens":0,"output_tokens":169,"reasoning_output_tokens":153},"cost":{"currency":"USD","total_usd":0.0086,"input_uncached_usd":0.00353,"input_cached_usd":0.0,"output_usd":0.00507,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.418049542000517}
{"item_id":"senseval2.d001.s078.t002","gold_sense_keys":["emerge%2:30:01::"],"candidates":[{"index":1,"sense_key":"emerge%2:42:00::","synset_id":"come_forth.v.02"},{"index":2,"sense_key":"emerge%2:38:00::","synset_id":"emerge.v.04"},{"index":3,"sense_key":"emerge%2:30:01::","synset_id":"emerge.v.03"},{"index":4,"sense_key":"emerge%2:30:02::","synset_id":"issue.v.04"},{"index":5,"sense_key":"emerge%2:30:00::","synset_id":"emerge.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"emerge%2:30:01::","call_ids":["senseval2.d001.s078.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"emerge%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.0046700000000000005,"input_uncached_usd":0.0029600000000000004,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6007200420135632}
{"item_id":"senseval2.d001.s078.t004","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"},{"index":2,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"},{"index":3,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"},{"index":4,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"},{"index":5,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"involved%3:00:00::","call_ids":["senseval2.d001.s078.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.0061200000000000004,"input_uncached_usd":0.00318,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3689399580471218}
{"item_id":"senseval2.d001.s078.t005","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":2,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":3,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":4,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s078.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":571,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.004325000000000001,"input_uncached_usd":0.0028550000000000003,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7736889999359846}
{"item_id":"senseval2.d001.s078.t006","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":2,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":3,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s078.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.004575,"input_uncached_usd":0.0029850000000000002,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":76.33432787505444}
{"item_id":"senseval2.d001.s078.t007","gold_sense_keys":["too%4:02:01::"],"candidates":[{"index":1,"sense_key":"too%4:02:01::","synset_id":"besides.r.02"},{"index":2,"sense_key":"too%4:02:00::","synset_id":"excessively.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"too%4:02:01::","call_ids":["senseval2.d001.s078.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"too%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.0043950000000000005,"input_uncached_usd":0.0023850000000000004,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9454436249798164}
{"item_id":"senseval2.d001.s079.t001","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":2,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":3,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":4,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":5,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":6,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"},{"index":7,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":8,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":9,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":10,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":13,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":14,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":15,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":16,"sense_key":"find%2:38:10::","synset_id":"find.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"find%2:32:00::","call_ids":["senseval2.d001.s079.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"find%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1182,"cached_input_tokens":0,"output_tokens":434,"reasoning_output_tokens":418},"cost":{"currency":"USD","total_usd":0.018930000000000002,"input_uncached_usd":0.00591,"input_cached_usd":0.0,"output_usd":0.01302,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.277051625074819}
{"item_id":"senseval2.d001.s079.t002","gold_sense_keys":["breast%1:08:00::"],"candidates":[{"index":1,"sense_key":"breast%1:13:00::","synset_id":"breast.n.03"},{"index":2,"sense_key":"breast%1:08:01::","synset_id":"breast.n.01"},{"index":3,"sense_key":"breast%1:05:00::","synset_id":"breast.n.04"},{"index":4,"sense_key":"breast%1:08:00::","synset_id":"breast.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"breast%1:08:00::","call_ids":["senseval2.d001.s079.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"breast%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":550,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.00545,"input_uncached_usd":0.0027500000000000003,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6153917079791427}
{"item_id":"senseval2.d001.s079.t004","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":3,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s079.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":540,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":85},"cost":{"currency":"USD","total_usd":0.0057,"input_uncached_usd":0.0027,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.898147333995439}
{"item_id":"senseval2.d001.s079.t007","gold_sense_keys":["spot%1:15:01::"],"candidates":[{"index":1,"sense_key":"spot%1:23:00::","synset_id":"spot.n.10"},{"index":2,"sense_key":"spot%1:26:00::","synset_id":"touch.n.08"},{"index":3,"sense_key":"spot%1:04:01::","synset_id":"position.n.06"},{"index":4,"sense_key":"spot%1:04:00::","synset_id":"blot.n.02"},{"index":5,"sense_key":"spot%1:06:01::","synset_id":"spotlight.n.02"},{"index":6,"sense_key":"spot%1:07:02::","synset_id":"point.n.14"},{"index":7,"sense_key":"spot%1:07:01::","synset_id":"spot.n.05"},{"index":8,"sense_key":"spot%1:06:02::","synset_id":"spot.n.13"},{"index":9,"sense_key":"spot%1:07:00::","synset_id":"smudge.n.02"},{"index":10,"sense_key":"spot%1:10:00::","synset_id":"spot.n.06"},{"index":11,"sense_key":"spot%1:10:01::","synset_id":"spot.n.02"},{"index":12,"sense_key":"spot%1:10:02::","synset_id":"spot.n.11"},{"index":13,"sense_key":"spot%1:06:00::","synset_id":"spot.n.07"},{"index":14,"sense_key":"spot%1:15:01::","synset_id":"topographic_point.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"spot%1:15:01::","call_ids":["senseval2.d001.s079.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"spot%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":979,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.0075049999999999995,"input_uncached_usd":0.004895,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8181767500936985}
{"item_id":"senseval2.d001.s080.t001","gold_sense_keys":["say%2:32:01::","say%2:32:13::"],"candidates":[{"index":1,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":2,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":5,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":6,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":7,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":8,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":11,"sense_key":"say%2:32:07::","synset_id":"say.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d001.s080.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":868,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02018,"input_uncached_usd":0.00434,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.6228785830317065}
{"item_id":"senseval2.d001.s080.t004","gold_sense_keys":["often%4:02:00::","often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":2,"sense_key":"often%4:02:02::","synset_id":"often.r.03"},{"index":3,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"often%4:02:02::","call_ids":["senseval2.d001.s080.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"often%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":240,"reasoning_output_tokens":224},"cost":{"currency":"USD","total_usd":0.009680000000000001,"input_uncached_usd":0.00248,"input_cached_usd":0.0,"output_usd":0.0072,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.730554667068645}
{"item_id":"senseval2.d001.s080.t005","gold_sense_keys":["strike%2:35:02::"],"candidates":[{"index":1,"sense_key":"strike%2:38:02::","synset_id":"hit.v.12"},{"index":2,"sense_key":"strike%2:38:08::","synset_id":"assume.v.05"},{"index":3,"sense_key":"strike%2:39:00::","synset_id":"hit.v.09"},{"index":4,"sense_key":"strike%2:40:00::","synset_id":"fall_upon.v.01"},{"index":5,"sense_key":"strike%2:35:03::","synset_id":"strike.v.10"},{"index":6,"sense_key":"strike%2:42:00::","synset_id":"fall.v.08"},{"index":7,"sense_key":"strike%2:35:10::","synset_id":"strickle.v.02"},{"index":8,"sense_key":"strike%2:35:00::","synset_id":"strike.v.01"},{"index":9,"sense_key":"strike%2:41:00::","synset_id":"strike.v.07"},{"index":10,"sense_key":"strike%2:35:09::","synset_id":"strike.v.20"},{"index":11,"sense_key":"strike%2:36:02::","synset_id":"mint.v.01"},{"index":12,"sense_key":"strike%2:32:00::","synset_id":"strike.v.05"},{"index":13,"sense_key":"strike%2:30:00::","synset_id":"strike.v.11"},{"index":14,"sense_key":"strike%2:31:00::","synset_id":"strike.v.21"},{"index":15,"sense_key":"strike%2:33:00::","synset_id":"strike.v.04"},{"index":16,"sense_key":"strike%2:37:00::","synset_id":"affect.v.05"},{"index":17,"sense_key":"strike%2:41:02::","synset_id":"come_to.v.03"},{"index":18,"sense_key":"strike%2:35:02::","synset_id":"hit.v.05"},{"index":19,"sense_key":"strike%2:36:00::","synset_id":"strike.v.13"},{"index":20,"sense_key":"strike%2:35:08::","synset_id":"strike.v.14"},{"index":21,"sense_key":"strike%2:35:01::","synset_id":"hit.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":18,"chosen_sense_key":"strike%2:35:02::","call_ids":["senseval2.d001.s080.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":18,"predicted_sense_key":"strike%2:35:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1349,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.009115,"input_uncached_usd":0.006745,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.980535125010647}
{"item_id":"senseval2.d001.s080.t007","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"},{"index":4,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":5,"sense_key":"member%1:18:00::","synset_id":"member.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"member%1:18:00::","call_ids":["senseval2.d001.s080.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"member%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":73},"cost":{"currency":"USD","total_usd":0.00585,"input_uncached_usd":0.00318,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.074019792024046}
{"item_id":"senseval2.d001.s080.t008","gold_sense_keys":["certain%5:00:00:definite:00"],"candidates":[{"index":1,"sense_key":"certain%5:00:00:definite:00","synset_id":"certain.s.01"},{"index":2,"sense_key":"certain%3:00:05::","synset_id":"sealed.a.01"},{"index":3,"sense_key":"certain%3:00:01::","synset_id":"certain.a.03"},{"index":4,"sense_key":"certain%5:00:00:careful:00","synset_id":"certain.s.03"},{"index":5,"sense_key":"certain%3:00:02::","synset_id":"certain.a.02"},{"index":6,"sense_key":"certain%5:00:00:reliable:00","synset_id":"certain.s.02"},{"index":7,"sense_key":"certain%3:00:03::","synset_id":"certain.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"certain%5:00:00:definite:00","call_ids":["senseval2.d001.s080.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"certain%5:00:00:definite:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.005065,"input_uncached_usd":0.0035350000000000004,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5803811250953004}
{"item_id":"senseval2.d001.s080.t009","gold_sense_keys":["family%1:14:01::"],"candidates":[{"index":1,"sense_key":"family%1:18:00::","synset_id":"kin.n.01"},{"index":2,"sense_key":"family%1:14:02::","synset_id":"family.n.01"},{"index":3,"sense_key":"family%1:14:00::","synset_id":"family.n.02"},{"index":4,"sense_key":"family%1:14:04::","synset_id":"class.n.01"},{"index":5,"sense_key":"family%1:14:01::","synset_id":"family.n.04"},{"index":6,"sense_key":"family%1:14:06::","synset_id":"syndicate.n.01"},{"index":7,"sense_key":"family%1:14:03::","synset_id":"family.n.06"},{"index":8,"sense_key":"family%1:14:05::","synset_id":"family.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"family%1:14:01::","call_ids":["senseval2.d001.s080.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"family%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.006255,"input_uncached_usd":0.003765,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.071617166977376}
{"item_id":"senseval2.d001.s080.t011","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"},{"index":2,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"},{"index":3,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s080.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.004505,"input_uncached_usd":0.002435,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3058362499577925}
{"item_id":"senseval2.d001.s080.t012","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s080.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.0045850000000000005,"input_uncached_usd":0.0023650000000000003,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8078990830108523}
{"item_id":"senseval2.d001.s080.t013","gold_sense_keys":["form%1:07:03::"],"candidates":[{"index":1,"sense_key":"form%1:07:03::","synset_id":"form.n.11"},{"index":2,"sense_key":"form%1:10:00::","synset_id":"form.n.01"},{"index":3,"sense_key":"form%1:06:00::","synset_id":"mannequin.n.02"},{"index":4,"sense_key":"form%1:26:00::","synset_id":"phase.n.02"},{"index":5,"sense_key":"form%1:10:01::","synset_id":"form.n.08"},{"index":6,"sense_key":"form%1:10:02::","synset_id":"form.n.10"},{"index":7,"sense_key":"form%1:14:02::","synset_id":"form.n.09"},{"index":8,"sense_key":"form%1:07:01::","synset_id":"shape.n.01"},{"index":9,"sense_key":"form%1:03:00::","synset_id":"shape.n.02"},{"index":10,"sense_key":"form%1:06:01::","synset_id":"form.n.16"},{"index":11,"sense_key":"form%1:09:00::","synset_id":"form.n.03"},{"index":12,"sense_key":"form%1:07:02::","synset_id":"form.n.14"},{"index":13,"sense_key":"form%1:09:01::","synset_id":"kind.n.01"},{"index":14,"sense_key":"form%1:14:00::","synset_id":"class.n.02"},{"index":15,"sense_key":"form%1:08:00::","synset_id":"human_body.n.01"},{"index":16,"sense_key":"form%1:07:00::","synset_id":"form.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"form%1:07:03::","call_ids":["senseval2.d001.s080.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"form%1:07:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1117,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.021425,"input_uncached_usd":0.0055850000000000006,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":16.390511583071202}
{"item_id":"senseval2.d001.s080.t014","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":4,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d001.s080.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.004685,"input_uncached_usd":0.002915,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6871454999782145}
{"item_id":"senseval2.d001.s080.t015","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":2,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":5,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s080.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":41,"reasoning_output_tokens":25},"cost":{"currency":"USD","total_usd":0.004155,"input_uncached_usd":0.002925,"input_cached_usd":0.0,"output_usd":0.00123,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6665666659828275}
{"item_id":"senseval2.d001.s081.t002","gold_sense_keys":["implicate%2:42:01::"],"candidates":[{"index":1,"sense_key":"implicate%2:42:01::","synset_id":"implicate.v.01"},{"index":2,"sense_key":"implicate%2:42:00::","synset_id":"entail.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"implicate%2:42:01::","call_ids":["senseval2.d001.s081.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"implicate%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":176,"reasoning_output_tokens":160},"cost":{"currency":"USD","total_usd":0.007660000000000001,"input_uncached_usd":0.00238,"input_cached_usd":0.0,"output_usd":0.00528,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.808279083110392}
{"item_id":"senseval2.d001.s082.t000","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":2,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":3,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"},{"index":4,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":5,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:10:00::","synset_id":"report.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"report%1:10:03::","call_ids":["senseval2.d001.s082.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":676,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":133},"cost":{"currency":"USD","total_usd":0.007850000000000001,"input_uncached_usd":0.00338,"input_cached_usd":0.0,"output_usd":0.00447,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.078031458077021}
{"item_id":"senseval2.d001.s082.t001","gold_sense_keys":["last%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"last%3:00:00::","synset_id":"last.a.02"},{"index":2,"sense_key":"last%5:00:00:closing:00","synset_id":"concluding.s.01"},{"index":3,"sense_key":"last%5:00:00:past:00","synset_id":"last.s.01"},{"index":4,"sense_key":"last%5:00:00:worst:00","synset_id":"last.s.08"},{"index":5,"sense_key":"last%5:00:00:unlikely:00","synset_id":"last.s.03"},{"index":6,"sense_key":"last%5:00:00:ultimate:00","synset_id":"final.s.02"},{"index":7,"sense_key":"last%5:00:00:unalterable:00","synset_id":"final.s.03"},{"index":8,"sense_key":"last%5:00:00:dying:00","synset_id":"last.s.04"},{"index":9,"sense_key":"last%5:00:00:high:02","synset_id":"last.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"last%5:00:00:past:00","call_ids":["senseval2.d001.s082.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"last%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":764,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00574,"input_uncached_usd":0.0038200000000000005,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.831123459036462}
{"item_id":"senseval2.d001.s082.t002","gold_sense_keys":["week%1:28:02::"],"candidates":[{"index":1,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":2,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"week%1:28:02::","call_ids":["senseval2.d001.s082.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"week%1:28:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":276,"reasoning_output_tokens":261},"cost":{"currency":"USD","total_usd":0.010575000000000001,"input_uncached_usd":0.002295,"input_cached_usd":0.0,"output_usd":0.008280000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.892457625013776}
{"item_id":"senseval2.d001.s082.t004","gold_sense_keys":["say%2:32:01::"],"candidates":[{"index":1,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":2,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":3,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":4,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":5,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":6,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":9,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":10,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":11,"sense_key":"say%2:32:15::","synset_id":"say.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d001.s082.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":492,"reasoning_output_tokens":476},"cost":{"currency":"USD","total_usd":0.018930000000000002,"input_uncached_usd":0.00417,"input_cached_usd":0.0,"output_usd":0.01476,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.975789499934763}
{"item_id":"senseval2.d001.s082.t005","gold_sense_keys":["about%4:02:00::"],"candidates":[{"index":1,"sense_key":"about%4:02:01::","synset_id":"about.r.03"},{"index":2,"sense_key":"about%4:02:00::","synset_id":"approximately.r.01"},{"index":3,"sense_key":"about%4:02:02::","synset_id":"about.r.05"},{"index":4,"sense_key":"about%4:02:06::","synset_id":"about.r.02"},{"index":5,"sense_key":"about%4:02:05::","synset_id":"about.r.07"},{"index":6,"sense_key":"about%4:02:03::","synset_id":"about.r.06"},{"index":7,"sense_key":"about%4:02:04::","synset_id":"about.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"about%4:02:00::","call_ids":["senseval2.d001.s082.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"about%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":674,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00508,"input_uncached_usd":0.00337,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2066472079604864}
{"item_id":"senseval2.d001.s082.t006","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":2,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":3,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":6,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s082.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.00453,"input_uncached_usd":0.0031200000000000004,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2880008750362322}
{"item_id":"senseval2.d001.s082.t007","gold_sense_keys":["take%2:30:00::"],"candidates":[{"index":1,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":2,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":3,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":4,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":5,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":6,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":7,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":8,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":11,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":12,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":13,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":14,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":15,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":16,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":17,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":18,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":19,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":20,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":21,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"},{"index":22,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":23,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":24,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":25,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":26,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":27,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":28,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":29,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":30,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":31,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":32,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":33,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":34,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":35,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":36,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":37,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":38,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":39,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":40,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":41,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":42,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":31,"chosen_sense_key":"take%2:30:00::","call_ids":["senseval2.d001.s082.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":31,"predicted_sense_key":"take%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2214,"cached_input_tokens":0,"output_tokens":177,"reasoning_output_tokens":161},"cost":{"currency":"USD","total_usd":0.016380000000000002,"input_uncached_usd":0.011070000000000002,"input_cached_usd":0.0,"output_usd":0.0053100000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.365796083002351}
{"item_id":"senseval2.d001.s082.t010","gold_sense_keys":["tissue%1:08:00::"],"candidates":[{"index":1,"sense_key":"tissue%1:27:02::","synset_id":"tissue.n.02"},{"index":2,"sense_key":"tissue%1:08:00::","synset_id":"tissue.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tissue%1:08:00::","call_ids":["senseval2.d001.s082.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tissue%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.00366,"input_uncached_usd":0.0020700000000000002,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.209654208039865}
{"item_id":"senseval2.d001.s082.t011","gold_sense_keys":["test%2:32:08::"],"candidates":[{"index":1,"sense_key":"test%2:32:00::","synset_id":"quiz.v.01"},{"index":2,"sense_key":"test%2:41:01::","synset_id":"screen.v.01"},{"index":3,"sense_key":"test%2:41:00::","synset_id":"test.v.01"},{"index":4,"sense_key":"test%2:42:00::","synset_id":"test.v.04"},{"index":5,"sense_key":"test%2:31:06::","synset_id":"test.v.07"},{"index":6,"sense_key":"test%2:32:08::","synset_id":"test.v.06"},{"index":7,"sense_key":"test%2:33:00::","synset_id":"test.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"test%2:32:08::","call_ids":["senseval2.d001.s082.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"test%2:32:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":247,"reasoning_output_tokens":231},"cost":{"currency":"USD","total_usd":0.010520000000000002,"input_uncached_usd":0.0031100000000000004,"input_cached_usd":0.0,"output_usd":0.00741,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.893262500059791}
{"item_id":"senseval2.d001.s082.t012","gold_sense_keys":["miss%2:42:00::"],"candidates":[{"index":1,"sense_key":"miss%2:37:00::","synset_id":"miss.v.02"},{"index":2,"sense_key":"miss%2:31:00::","synset_id":"neglect.v.01"},{"index":3,"sense_key":"miss%2:42:02::","synset_id":"miss.v.08"},{"index":4,"sense_key":"miss%2:38:00::","synset_id":"miss.v.05"},{"index":5,"sense_key":"miss%2:32:00::","synset_id":"miss.v.09"},{"index":6,"sense_key":"miss%2:42:01::","synset_id":"miss.v.03"},{"index":7,"sense_key":"miss%2:42:00::","synset_id":"miss.v.06"},{"index":8,"sense_key":"miss%2:39:00::","synset_id":"miss.v.01"},{"index":9,"sense_key":"miss%2:35:00::","synset_id":"miss.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"miss%2:42:00::","call_ids":["senseval2.d001.s082.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"miss%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.007755,"input_uncached_usd":0.003435,"input_cached_usd":0.0,"output_usd":0.00432,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3040127080166712}
{"item_id":"senseval2.d001.s083.t000","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":2,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":3,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":4,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":5,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":6,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"},{"index":7,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"report%1:10:03::","call_ids":["senseval2.d001.s083.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":657,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019125,"input_uncached_usd":0.003285,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.9311202499084175}
{"item_id":"senseval2.d001.s083.t003","gold_sense_keys":["yet%4:02:02::"],"candidates":[{"index":1,"sense_key":"yet%4:02:05::","synset_id":"so_far.r.03"},{"index":2,"sense_key":"yet%4:02:06::","synset_id":"however.r.01"},{"index":3,"sense_key":"yet%4:02:04::","synset_id":"so_far.r.01"},{"index":4,"sense_key":"yet%4:02:03::","synset_id":"even.r.03"},{"index":5,"sense_key":"yet%4:02:02::","synset_id":"yet.r.01"},{"index":6,"sense_key":"yet%4:02:00::","synset_id":"yet.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"yet%4:02:04::","call_ids":["senseval2.d001.s083.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"yet%4:02:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":218,"reasoning_output_tokens":203},"cost":{"currency":"USD","total_usd":0.009899999999999999,"input_uncached_usd":0.00336,"input_cached_usd":0.0,"output_usd":0.00654,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.895415290957317}
{"item_id":"senseval2.d001.s083.t007","gold_sense_keys":["tissue%1:08:00::"],"candidates":[{"index":1,"sense_key":"tissue%1:27:02::","synset_id":"tissue.n.02"},{"index":2,"sense_key":"tissue%1:08:00::","synset_id":"tissue.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tissue%1:08:00::","call_ids":["senseval2.d001.s083.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tissue%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":395,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.0035050000000000003,"input_uncached_usd":0.001975,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.188441083999351}
{"item_id":"senseval2.d001.s083.t008","gold_sense_keys":["take%2:30:00::"],"candidates":[{"index":1,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":2,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":3,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":6,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":7,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":8,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":9,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":10,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":11,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":12,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":13,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":14,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":15,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":16,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":17,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"},{"index":18,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":19,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":20,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":21,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":22,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":23,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":24,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":25,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":26,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":27,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":28,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":29,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":30,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":31,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":32,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":33,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":34,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":35,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":36,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":37,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":38,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":39,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":40,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":41,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":42,"sense_key":"take%2:31:07::","synset_id":"take.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"take%2:30:00::","call_ids":["senseval2.d001.s083.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"take%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2195,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":130},"cost":{"currency":"USD","total_usd":0.015325000000000002,"input_uncached_usd":0.010975,"input_cached_usd":0.0,"output_usd":0.00435,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5703229159116745}
{"item_id":"senseval2.d001.s083.t010","gold_sense_keys":["brain%1:08:00::"],"candidates":[{"index":1,"sense_key":"brain%1:08:00::","synset_id":"brain.n.01"},{"index":2,"sense_key":"brain%1:13:00::","synset_id":"brain.n.05"},{"index":3,"sense_key":"brain%1:09:01::","synset_id":"mind.n.01"},{"index":4,"sense_key":"brain%1:09:00::","synset_id":"brain.n.02"},{"index":5,"sense_key":"brain%1:18:00::","synset_id":"genius.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brain%1:08:00::","call_ids":["senseval2.d001.s083.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brain%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004625000000000001,"input_uncached_usd":0.0028250000000000003,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.785533499903977}
{"item_id":"senseval2.d001.s084.t000","gold_sense_keys":["at_the_same_time%4:02:05::"],"candidates":[{"index":1,"sense_key":"at_the_same_time%4:02:05::","synset_id":"concurrently.r.01"},{"index":2,"sense_key":"at_the_same_time%4:02:03::","synset_id":"simultaneously.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"at_the_same_time%4:02:05::","call_ids":["senseval2.d001.s084.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"at_the_same_time%4:02:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":96},"cost":{"currency":"USD","total_usd":0.00549,"input_uncached_usd":0.00216,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5845508750062436}
{"item_id":"senseval2.d001.s084.t001","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:01::","synset_id":"team.n.02"},{"index":2,"sense_key":"team%1:14:00::","synset_id":"team.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"team%1:14:00::","call_ids":["senseval2.d001.s084.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0038350000000000003,"input_uncached_usd":0.0019450000000000001,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2684851669473574}
{"item_id":"senseval2.d001.s084.t002","gold_sense_keys":["rush%2:30:00::"],"candidates":[{"index":1,"sense_key":"rush%2:36:00::","synset_id":"induce.v.03"},{"index":2,"sense_key":"rush%2:30:00::","synset_id":"rush.v.04"},{"index":3,"sense_key":"rush%2:38:01::","synset_id":"rush.v.05"},{"index":4,"sense_key":"rush%2:30:01::","synset_id":"rush.v.03"},{"index":5,"sense_key":"rush%2:38:00::","synset_id":"rush.v.01"},{"index":6,"sense_key":"rush%2:33:00::","synset_id":"rush.v.02"},{"index":7,"sense_key":"rush%2:38:10::","synset_id":"race.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"rush%2:30:00::","call_ids":["senseval2.d001.s084.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"rush%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.006095,"input_uncached_usd":0.0030050000000000003,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.432766209007241}
{"item_id":"senseval2.d001.s084.t003","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":2,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":3,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":4,"sense_key":"other%3:00:00::","synset_id":"other.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s084.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":507,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.004635,"input_uncached_usd":0.0025350000000000004,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.424586958019063}
{"item_id":"senseval2.d001.s085.t000","gold_sense_keys":["hope%2:31:00::"],"candidates":[{"index":1,"sense_key":"hope%2:37:00::","synset_id":"hope.v.02"},{"index":2,"sense_key":"hope%2:37:01::","synset_id":"hope.v.01"},{"index":3,"sense_key":"hope%2:31:00::","synset_id":"hope.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"hope%2:31:00::","call_ids":["senseval2.d001.s085.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"hope%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.005035,"input_uncached_usd":0.0023350000000000003,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1359065419528633}
{"item_id":"senseval2.d001.s085.t004","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"},{"index":2,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"},{"index":3,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"},{"index":4,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"},{"index":5,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"involved%3:00:00::","call_ids":["senseval2.d001.s085.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.00515,"input_uncached_usd":0.00287,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3782513750484213}
{"item_id":"senseval2.d001.s086.t001","gold_sense_keys":["close%3:00:01::","close%3:00:05::"],"candidates":[{"index":1,"sense_key":"close%5:00:00:confined:00","synset_id":"close.s.04"},{"index":2,"sense_key":"close%5:00:00:accurate:00","synset_id":"close.s.02"},{"index":3,"sense_key":"close%5:00:00:stingy:00","synset_id":"cheeseparing.s.01"},{"index":4,"sense_key":"close%5:00:00:restrained:00","synset_id":"close.s.07"},{"index":5,"sense_key":"close%5:00:00:short:01","synset_id":"close.s.10"},{"index":6,"sense_key":"close%5:00:00:tight:01","synset_id":"close.s.09"},{"index":7,"sense_key":"close%3:00:02::","synset_id":"close.a.02"},{"index":8,"sense_key":"close%5:00:00:unventilated:00","synset_id":"airless.s.01"},{"index":9,"sense_key":"close%5:00:00:uncommunicative:00","synset_id":"close.s.12"},{"index":10,"sense_key":"close%3:00:05::","synset_id":"near.a.01"},{"index":11,"sense_key":"close%5:00:00:equal:00","synset_id":"close.s.03"},{"index":12,"sense_key":"close%5:00:00:private:00","synset_id":"close.s.08"},{"index":13,"sense_key":"close%3:00:01::","synset_id":"close.a.01"},{"index":14,"sense_key":"close%5:00:00:careful:00","synset_id":"close.s.01"},{"index":15,"sense_key":"close%5:00:00:fine:00","synset_id":"close.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"close%3:00:05::","call_ids":["senseval2.d001.s086.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"close%3:00:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":963,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020655,"input_uncached_usd":0.004815000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.076382583007216}
{"item_id":"senseval2.d001.s086.t002","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"},{"index":2,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":3,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":4,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":5,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":6,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":9,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":10,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":11,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":14,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":15,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":16,"sense_key":"find%2:40:00::","synset_id":"find.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"find%2:36:00::","call_ids":["senseval2.d001.s086.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"find%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1123,"cached_input_tokens":0,"output_tokens":1135,"reasoning_output_tokens":1110},"cost":{"currency":"USD","total_usd":0.039665000000000006,"input_uncached_usd":0.005615,"input_cached_usd":0.0,"output_usd":0.034050000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":19.541250042035244}
{"item_id":"senseval2.d001.s086.t004","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"},{"index":2,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"},{"index":3,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"},{"index":4,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"},{"index":5,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"involved%3:00:00::","call_ids":["senseval2.d001.s086.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.00453,"input_uncached_usd":0.00285,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7961608750047162}
{"item_id":"senseval2.d001.s086.t005","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":2,"sense_key":"type%1:06:00::","synset_id":"type.n.06"},{"index":3,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":4,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":5,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":6,"sense_key":"type%1:10:01::","synset_id":"type.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"type%1:09:00::","call_ids":["senseval2.d001.s086.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.00437,"input_uncached_usd":0.00284,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.866793749970384}
{"item_id":"senseval2.d001.s087.t000","gold_sense_keys":["believe%2:31:04::"],"candidates":[{"index":1,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":2,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"},{"index":3,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"},{"index":4,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"},{"index":5,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"believe%2:31:04::","call_ids":["senseval2.d001.s087.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"believe%2:31:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":1564,"reasoning_output_tokens":1535},"cost":{"currency":"USD","total_usd":0.04979,"input_uncached_usd":0.00287,"input_cached_usd":0.0,"output_usd":0.04692,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":25.63241304201074}
{"item_id":"senseval2.d001.s087.t001","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"},{"index":2,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":3,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":4,"sense_key":"people%1:14:00::","synset_id":"people.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval2.d001.s087.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.0049700000000000005,"input_uncached_usd":0.00266,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.943789790966548}
{"item_id":"senseval2.d001.s087.t002","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"},{"index":2,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"},{"index":3,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s087.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":42,"reasoning_output_tokens":27},"cost":{"currency":"USD","total_usd":0.003535,"input_uncached_usd":0.002275,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7098499579587951}
{"item_id":"senseval2.d001.s087.t006","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"},{"index":3,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":4,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s087.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.00516,"input_uncached_usd":0.00252,"input_cached_usd":0.0,"output_usd":0.00264,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9643360410118476}
{"item_id":"senseval2.d001.s087.t008","gold_sense_keys":["especially%4:02:00::"],"candidates":[{"index":1,"sense_key":"especially%4:02:00::","synset_id":"particularly.r.01"},{"index":2,"sense_key":"especially%4:02:01::","synset_id":"specially.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"especially%4:02:00::","call_ids":["senseval2.d001.s087.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"especially%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.004045,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.5928622499341145}
{"item_id":"senseval2.d001.s088.t001","gold_sense_keys":["report%2:32:04::"],"candidates":[{"index":1,"sense_key":"report%2:32:02::","synset_id":"report.v.06"},{"index":2,"sense_key":"report%2:32:13::","synset_id":"report.v.05"},{"index":3,"sense_key":"report%2:32:00::","synset_id":"report.v.01"},{"index":4,"sense_key":"report%2:32:03::","synset_id":"report.v.03"},{"index":5,"sense_key":"report%2:32:04::","synset_id":"report.v.02"},{"index":6,"sense_key":"report%2:32:01::","synset_id":"report.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"report%2:32:04::","call_ids":["senseval2.d001.s088.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"report%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.005385,"input_uncached_usd":0.002925,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7098666250240058}
{"item_id":"senseval2.d001.s088.t005","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"},{"index":2,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"},{"index":3,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"},{"index":4,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"},{"index":5,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"involved%3:00:00::","call_ids":["senseval2.d001.s088.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.005335,"input_uncached_usd":0.0028450000000000003,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.909542000037618}
{"item_id":"senseval2.d001.s088.t008","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":4,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s088.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.00441,"input_uncached_usd":0.00252,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.298292582971044}
{"item_id":"senseval2.d001.s088.t009","gold_sense_keys":["more%4:02:00::"],"candidates":[{"index":1,"sense_key":"more%4:02:01::","synset_id":"more.r.02"},{"index":2,"sense_key":"more%4:02:00::","synset_id":"more.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"more%4:02:00::","call_ids":["senseval2.d001.s088.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"more%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.004565,"input_uncached_usd":0.002075,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1100892079994082}
{"item_id":"senseval2.d001.s088.t010","gold_sense_keys":["common%3:00:01::","common%5:00:00:familiar:02"],"candidates":[{"index":1,"sense_key":"common%3:00:02::","synset_id":"common.a.01"},{"index":2,"sense_key":"common%5:00:00:lowborn:00","synset_id":"common.s.04"},{"index":3,"sense_key":"common%5:00:00:inferior:02","synset_id":"coarse.s.02"},{"index":4,"sense_key":"common%5:00:00:ordinary:00","synset_id":"common.s.07"},{"index":5,"sense_key":"common%5:00:00:informal:02","synset_id":"common.s.03"},{"index":6,"sense_key":"common%5:00:00:familiar:02","synset_id":"common.s.02"},{"index":7,"sense_key":"common%5:00:00:unrefined:01","synset_id":"coarse.s.01"},{"index":8,"sense_key":"common%5:00:00:shared:00","synset_id":"common.s.01"},{"index":9,"sense_key":"common%3:00:01::","synset_id":"common.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"common%5:00:00:familiar:02","call_ids":["senseval2.d001.s088.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"common%5:00:00:familiar:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":267,"reasoning_output_tokens":251},"cost":{"currency":"USD","total_usd":0.011865,"input_uncached_usd":0.0038550000000000004,"input_cached_usd":0.0,"output_usd":0.00801,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.412351166945882}
{"item_id":"senseval2.d001.s088.t011","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":2,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":3,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":4,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":5,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s088.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":40,"reasoning_output_tokens":25},"cost":{"currency":"USD","total_usd":0.00385,"input_uncached_usd":0.00265,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2345792499836534}
{"item_id":"senseval2.d001.s089.t000","gold_sense_keys":["discovery%1:04:00::","discovery%1:09:00::","discovery%1:10:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"},{"index":2,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"discovery%1:10:00::","call_ids":["senseval2.d001.s089.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"discovery%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.03378,"input_uncached_usd":0.0024000000000000002,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":22.161330083967187}
{"item_id":"senseval2.d001.s089.t003","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":6,"sense_key":"only%4:02:05::","synset_id":"only.r.07"},{"index":7,"sense_key":"only%4:02:03::","synset_id":"only.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"only%4:02:02::","call_ids":["senseval2.d001.s089.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"only%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.005550000000000001,"input_uncached_usd":0.0031500000000000005,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.136617333977483}
{"item_id":"senseval2.d001.s089.t004","gold_sense_keys":["speculate%2:31:01::","speculate%2:32:00::"],"candidates":[{"index":1,"sense_key":"speculate%2:40:00::","synset_id":"speculate.v.04"},{"index":2,"sense_key":"speculate%2:32:00::","synset_id":"speculate.v.02"},{"index":3,"sense_key":"speculate%2:31:01::","synset_id":"speculate.v.01"},{"index":4,"sense_key":"speculate%2:31:00::","synset_id":"chew_over.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"speculate%2:31:01::","call_ids":["senseval2.d001.s089.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"speculate%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":1752,"reasoning_output_tokens":1718},"cost":{"currency":"USD","total_usd":0.055545000000000004,"input_uncached_usd":0.0029850000000000002,"input_cached_usd":0.0,"output_usd":0.05256,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":26.46223399997689}
{"item_id":"senseval2.d001.s090.t001","gold_sense_keys":["major%3:00:06::"],"candidates":[{"index":1,"sense_key":"major%3:00:06::","synset_id":"major.a.01"},{"index":2,"sense_key":"major%3:00:07::","synset_id":"major.a.06"},{"index":3,"sense_key":"major%3:00:01::","synset_id":"major.a.03"},{"index":4,"sense_key":"major%3:00:05::","synset_id":"major.a.04"},{"index":5,"sense_key":"major%3:00:02::","synset_id":"major.a.02"},{"index":6,"sense_key":"major%3:00:04::","synset_id":"major.a.05"},{"index":7,"sense_key":"major%5:00:00:senior:00","synset_id":"major.s.01"},{"index":8,"sense_key":"major%3:00:03::","synset_id":"major.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"major%3:00:06::","call_ids":["senseval2.d001.s090.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"major%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":306,"reasoning_output_tokens":291},"cost":{"currency":"USD","total_usd":0.012265000000000002,"input_uncached_usd":0.003085,"input_cached_usd":0.0,"output_usd":0.00918,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.294831000035629}
{"item_id":"senseval2.d001.s090.t004","gold_sense_keys":["unit%1:14:00::"],"candidates":[{"index":1,"sense_key":"unit%1:03:00::","synset_id":"whole.n.02"},{"index":2,"sense_key":"unit%1:24:00::","synset_id":"unit.n.02"},{"index":3,"sense_key":"unit%1:17:00::","synset_id":"unit.n.05"},{"index":4,"sense_key":"unit%1:14:00::","synset_id":"unit.n.03"},{"index":5,"sense_key":"unit%1:23:00::","synset_id":"unit_of_measurement.n.01"},{"index":6,"sense_key":"unit%1:09:00::","synset_id":"unit.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"unit%1:14:00::","call_ids":["senseval2.d001.s090.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"unit%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":657,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.006225,"input_uncached_usd":0.003285,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6599955830024555}
{"item_id":"senseval2.d001.s090.t005","gold_sense_keys":["collaborate%2:41:00::"],"candidates":[{"index":1,"sense_key":"collaborate%2:41:01::","synset_id":"collaborate.v.02"},{"index":2,"sense_key":"collaborate%2:41:00::","synset_id":"collaborate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"collaborate%2:41:00::","call_ids":["senseval2.d001.s090.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"collaborate%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":29,"reasoning_output_tokens":14},"cost":{"currency":"USD","total_usd":0.003135,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.00087,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.278051417088136}
{"item_id":"senseval2.d001.s090.t007","gold_sense_keys":["hunter%1:18:01::"],"candidates":[{"index":1,"sense_key":"hunter%1:18:00::","synset_id":"hunter.n.01"},{"index":2,"sense_key":"hunter%1:17:00::","synset_id":"orion.n.02"},{"index":3,"sense_key":"hunter%1:18:01::","synset_id":"hunter.n.02"},{"index":4,"sense_key":"hunter%1:06:00::","synset_id":"hunter.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"hunter%1:18:01::","call_ids":["senseval2.d001.s090.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"hunter%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.00388,"input_uncached_usd":0.0024100000000000002,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3755772080039605}
{"item_id":"senseval2.d001.s090.t008","gold_sense_keys":["turn%2:42:13::"],"candidates":[{"index":1,"sense_key":"turn%2:30:04::","synset_id":"change_state.v.01"},{"index":2,"sense_key":"turn%2:38:03::","synset_id":"turn.v.06"},{"index":3,"sense_key":"turn%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":4,"sense_key":"turn%2:36:00::","synset_id":"plow.v.01"},{"index":5,"sense_key":"turn%2:35:08::","synset_id":"turn.v.08"},{"index":6,"sense_key":"turn%2:38:00::","synset_id":"turn.v.01"},{"index":7,"sense_key":"turn%2:38:02::","synset_id":"turn.v.04"},{"index":8,"sense_key":"turn%2:35:11::","synset_id":"flex.v.05"},{"index":9,"sense_key":"turn%2:30:03::","synset_id":"turn.v.07"},{"index":10,"sense_key":"turn%2:38:14::","synset_id":"turn.v.20"},{"index":11,"sense_key":"turn%2:35:06::","synset_id":"turn.v.13"},{"index":12,"sense_key":"turn%2:30:09::","synset_id":"turn.v.14"},{"index":13,"sense_key":"turn%2:29:00::","synset_id":"twist.v.10"},{"index":14,"sense_key":"turn%2:40:13::","synset_id":"turn.v.18"},{"index":15,"sense_key":"turn%2:42:13::","synset_id":"turn.v.16"},{"index":16,"sense_key":"turn%2:38:06::","synset_id":"turn.v.11"},{"index":17,"sense_key":"turn%2:42:00::","synset_id":"become.v.02"},{"index":18,"sense_key":"turn%2:30:14::","synset_id":"turn.v.26"},{"index":19,"sense_key":"turn%2:38:04::","synset_id":"turn.v.09"},{"index":20,"sense_key":"turn%2:35:10::","synset_id":"turn.v.22"},{"index":21,"sense_key":"turn%2:38:01::","synset_id":"turn.v.10"},{"index":22,"sense_key":"turn%2:33:13::","synset_id":"turn.v.23"},{"index":23,"sense_key":"turn%2:38:13::","synset_id":"turn.v.19"},{"index":24,"sense_key":"turn%2:32:14::","synset_id":"call_on.v.01"},{"index":25,"sense_key":"turn%2:30:01::","synset_id":"sour.v.01"},{"index":26,"sense_key":"turn%2:41:10::","synset_id":"turn.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"turn%2:42:13::","call_ids":["senseval2.d001.s090.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"turn%2:42:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1478,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":130},"cost":{"currency":"USD","total_usd":0.011770000000000001,"input_uncached_usd":0.007390000000000001,"input_cached_usd":0.0,"output_usd":0.00438,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4513452500104904}
{"item_id":"senseval2.d001.s090.t010","gold_sense_keys":["discovery%1:10:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"},{"index":2,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":3,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":4,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"discovery%1:10:00::","call_ids":["senseval2.d001.s090.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"discovery%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01826,"input_uncached_usd":0.0024200000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.760109041933902}
{"item_id":"senseval2.d001.s090.t014","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":5,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"},{"index":6,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":7,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":8,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":9,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":10,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":11,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d001.s090.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":838,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.03557,"input_uncached_usd":0.00419,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":19.91152470803354}
{"item_id":"senseval2.d001.s091.t001","gold_sense_keys":["say%2:32:01::","say%2:32:13::"],"candidates":[{"index":1,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":2,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":3,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":4,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":5,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":6,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":7,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":8,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":11,"sense_key":"say%2:42:00::","synset_id":"read.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d001.s091.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":768,"reasoning_output_tokens":746},"cost":{"currency":"USD","total_usd":0.027235000000000002,"input_uncached_usd":0.004195,"input_cached_usd":0.0,"output_usd":0.02304,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":16.971944708027877}
{"item_id":"senseval2.d001.s091.t002","gold_sense_keys":["new%3:00:00::","new%3:00:09::","new%5:00:00:original:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":2,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":5,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"},{"index":6,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":7,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":8,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":9,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":10,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":11,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d001.s091.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":840,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020040000000000002,"input_uncached_usd":0.004200000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.102492375066504}
{"item_id":"senseval2.d001.s091.t005","gold_sense_keys":["slow%2:30:05::"],"candidates":[{"index":1,"sense_key":"slow%2:30:01::","synset_id":"slow.v.02"},{"index":2,"sense_key":"slow%2:30:05::","synset_id":"slow.v.03"},{"index":3,"sense_key":"slow%2:30:00::","synset_id":"decelerate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"slow%2:30:05::","call_ids":["senseval2.d001.s091.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"slow%2:30:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.0045249999999999995,"input_uncached_usd":0.002395,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.745511208078824}
{"item_id":"senseval2.d001.s091.t006","gold_sense_keys":["reverse%2:30:00::"],"candidates":[{"index":1,"sense_key":"reverse%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":2,"sense_key":"reverse%2:30:01::","synset_id":"turn_back.v.05"},{"index":3,"sense_key":"reverse%2:30:03::","synset_id":"invert.v.02"},{"index":4,"sense_key":"reverse%2:32:00::","synset_id":"revoke.v.02"},{"index":5,"sense_key":"reverse%2:31:00::","synset_id":"overrule.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reverse%2:30:00::","call_ids":["senseval2.d001.s091.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reverse%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.005355,"input_uncached_usd":0.0028350000000000003,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.766991415992379}
{"item_id":"senseval2.d001.s091.t008","gold_sense_keys":["growth%1:22:00::","growth%1:26:00::"],"candidates":[{"index":1,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":2,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":3,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":4,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":5,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":6,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"},{"index":7,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"growth%1:22:02::","call_ids":["senseval2.d001.s091.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"growth%1:22:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":3334,"reasoning_output_tokens":3283},"cost":{"currency":"USD","total_usd":0.10326500000000001,"input_uncached_usd":0.003245,"input_cached_usd":0.0,"output_usd":0.10002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":52.86059979198035}
{"item_id":"senseval2.d001.s091.t009","gold_sense_keys":["base%2:31:00::"],"candidates":[{"index":1,"sense_key":"base%2:34:00::","synset_id":"free-base.v.01"},{"index":2,"sense_key":"base%2:31:00::","synset_id":"establish.v.08"},{"index":3,"sense_key":"base%2:42:04::","synset_id":"base.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"base%2:31:00::","call_ids":["senseval2.d001.s091.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"base%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.004005,"input_uncached_usd":0.0023550000000000003,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0175050840480253}
{"item_id":"senseval2.d001.s091.t012","gold_sense_keys":["produce%2:36:02::","produce%2:36:03::"],"candidates":[{"index":1,"sense_key":"produce%2:36:00::","synset_id":"produce.v.02"},{"index":2,"sense_key":"produce%2:39:00::","synset_id":"produce.v.06"},{"index":3,"sense_key":"produce%2:36:03::","synset_id":"produce.v.03"},{"index":4,"sense_key":"produce%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"produce%2:36:05::","synset_id":"grow.v.07"},{"index":6,"sense_key":"produce%2:36:02::","synset_id":"produce.v.01"},{"index":7,"sense_key":"produce%2:39:01::","synset_id":"produce.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"produce%2:36:02::","call_ids":["senseval2.d001.s091.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"produce%2:36:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":190,"reasoning_output_tokens":174},"cost":{"currency":"USD","total_usd":0.009185,"input_uncached_usd":0.0034850000000000003,"input_cached_usd":0.0,"output_usd":0.0057,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.338901125011034}
{"item_id":"senseval2.d001.s092.t000","gold_sense_keys":["idea%1:09:01::"],"candidates":[{"index":1,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"},{"index":2,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"},{"index":3,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"},{"index":4,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"},{"index":5,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"idea%1:09:01::","call_ids":["senseval2.d001.s092.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"idea%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.00528,"input_uncached_usd":0.0029400000000000003,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0507087919395417}
{"item_id":"senseval2.d001.s092.t001","gold_sense_keys":["administer%2:29:00::"],"candidates":[{"index":1,"sense_key":"administer%2:40:00::","synset_id":"distribute.v.01"},{"index":2,"sense_key":"administer%2:29:00::","synset_id":"administer.v.04"},{"index":3,"sense_key":"administer%2:40:01::","synset_id":"administer.v.02"},{"index":4,"sense_key":"administer%2:41:01::","synset_id":"administer.v.05"},{"index":5,"sense_key":"administer%2:41:00::","synset_id":"administer.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"administer%2:29:00::","call_ids":["senseval2.d001.s092.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"administer%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00456,"input_uncached_usd":0.0027600000000000003,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9526934580644593}
{"item_id":"senseval2.d001.s092.t002","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["senseval2.d001.s092.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":431,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.003745,"input_uncached_usd":0.0021550000000000002,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3968488750979304}
{"item_id":"senseval2.d001.s092.t005","gold_sense_keys":["healthy%3:00:00::"],"candidates":[{"index":1,"sense_key":"healthy%5:00:00:reasonable:00","synset_id":"healthy.s.03"},{"index":2,"sense_key":"healthy%5:00:00:wholesome:00","synset_id":"healthy.s.02"},{"index":3,"sense_key":"healthy%5:00:00:considerable:00","synset_id":"goodly.s.01"},{"index":4,"sense_key":"healthy%3:00:00::","synset_id":"healthy.a.01"},{"index":5,"sense_key":"healthy%5:00:00:sound:00","synset_id":"healthy.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"healthy%3:00:00::","call_ids":["senseval2.d001.s092.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"healthy%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":103},"cost":{"currency":"USD","total_usd":0.006495,"input_uncached_usd":0.002925,"input_cached_usd":0.0,"output_usd":0.0035700000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.739674041979015}
{"item_id":"senseval2.d001.s092.t006","gold_sense_keys":["version%1:09:01::"],"candidates":[{"index":1,"sense_key":"version%1:10:00::","synset_id":"translation.n.01"},{"index":2,"sense_key":"version%1:09:01::","synset_id":"version.n.02"},{"index":3,"sense_key":"version%1:10:02::","synset_id":"adaptation.n.01"},{"index":4,"sense_key":"version%1:04:00::","synset_id":"version.n.06"},{"index":5,"sense_key":"version%1:10:01::","synset_id":"version.n.01"},{"index":6,"sense_key":"version%1:09:00::","synset_id":"interpretation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"version%1:09:01::","call_ids":["senseval2.d001.s092.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"version%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":593,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.004615,"input_uncached_usd":0.002965,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.881579375010915}
{"item_id":"senseval2.d001.s092.t007","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s092.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00405,"input_uncached_usd":0.00216,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.227779666078277}
{"item_id":"senseval2.d001.s093.t000","gold_sense_keys":["possible%3:00:00::"],"candidates":[{"index":1,"sense_key":"possible%3:00:00::","synset_id":"possible.a.01"},{"index":2,"sense_key":"possible%3:00:04::","synset_id":"potential.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"possible%3:00:00::","call_ids":["senseval2.d001.s093.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"possible%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":85},"cost":{"currency":"USD","total_usd":0.005095000000000001,"input_uncached_usd":0.002095,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0248158749891445}
{"item_id":"senseval2.d001.s093.t001","gold_sense_keys":["replace%2:30:00::"],"candidates":[{"index":1,"sense_key":"replace%2:30:00::","synset_id":"replace.v.01"},{"index":2,"sense_key":"replace%2:35:00::","synset_id":"replace.v.03"},{"index":3,"sense_key":"replace%2:41:00::","synset_id":"supplant.v.01"},{"index":4,"sense_key":"replace%2:40:00::","synset_id":"substitute.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"replace%2:30:00::","call_ids":["senseval2.d001.s093.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"replace%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":327,"reasoning_output_tokens":312},"cost":{"currency":"USD","total_usd":0.012750000000000001,"input_uncached_usd":0.0029400000000000003,"input_cached_usd":0.0,"output_usd":0.009810000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.572838500025682}
{"item_id":"senseval2.d001.s093.t004","gold_sense_keys":["healthy%3:00:00::"],"candidates":[{"index":1,"sense_key":"healthy%5:00:00:considerable:00","synset_id":"goodly.s.01"},{"index":2,"sense_key":"healthy%5:00:00:wholesome:00","synset_id":"healthy.s.02"},{"index":3,"sense_key":"healthy%3:00:00::","synset_id":"healthy.a.01"},{"index":4,"sense_key":"healthy%5:00:00:sound:00","synset_id":"healthy.s.01"},{"index":5,"sense_key":"healthy%5:00:00:reasonable:00","synset_id":"healthy.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"healthy%3:00:00::","call_ids":["senseval2.d001.s093.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"healthy%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":115},"cost":{"currency":"USD","total_usd":0.006875000000000001,"input_uncached_usd":0.002945,"input_cached_usd":0.0,"output_usd":0.00393,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.55117891705595}
{"item_id":"senseval2.d001.s093.t005","gold_sense_keys":["version%1:09:01::"],"candidates":[{"index":1,"sense_key":"version%1:10:01::","synset_id":"version.n.01"},{"index":2,"sense_key":"version%1:04:00::","synset_id":"version.n.06"},{"index":3,"sense_key":"version%1:10:00::","synset_id":"translation.n.01"},{"index":4,"sense_key":"version%1:10:02::","synset_id":"adaptation.n.01"},{"index":5,"sense_key":"version%1:09:00::","synset_id":"interpretation.n.01"},{"index":6,"sense_key":"version%1:09:01::","synset_id":"version.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"version%1:09:01::","call_ids":["senseval2.d001.s093.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"version%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.005560000000000001,"input_uncached_usd":0.0029800000000000004,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.20777720795013}
{"item_id":"senseval2.d001.s093.t006","gold_sense_keys":["come_close%2:41:00::"],"candidates":[{"index":1,"sense_key":"come_close%2:42:00::","synset_id":"approximate.v.01"},{"index":2,"sense_key":"come_close%2:41:00::","synset_id":"come_close.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"come_close%2:41:00::","call_ids":["senseval2.d001.s093.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"come_close%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.003855,"input_uncached_usd":0.002085,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5927775839809328}
{"item_id":"senseval2.d001.s094.t001","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":2,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":3,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":4,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":8,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":9,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":10,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":11,"sense_key":"say%2:32:00::","synset_id":"state.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d001.s094.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":796,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.035359999999999996,"input_uncached_usd":0.00398,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":20.737774333101697}
{"item_id":"senseval2.d001.s094.t002","gold_sense_keys":["witness%2:39:01::"],"candidates":[{"index":1,"sense_key":"witness%2:39:00::","synset_id":"witness.v.01"},{"index":2,"sense_key":"witness%2:39:01::","synset_id":"witness.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"witness%2:39:01::","call_ids":["senseval2.d001.s094.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"witness%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":397,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.003905,"input_uncached_usd":0.0019850000000000002,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6072597909951583}
{"item_id":"senseval2.d001.s094.t003","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":4,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"discovery%1:04:00::","call_ids":["senseval2.d001.s094.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"discovery%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":321,"reasoning_output_tokens":305},"cost":{"currency":"USD","total_usd":0.011845,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.00963,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.316536042024381}
{"item_id":"senseval2.d001.s094.t004","gold_sense_keys":["most%4:02:00::"],"candidates":[{"index":1,"sense_key":"most%4:02:01::","synset_id":"most.r.02"},{"index":2,"sense_key":"most%4:02:02::","synset_id":"about.r.07"},{"index":3,"sense_key":"most%4:02:00::","synset_id":"most.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"most%4:02:00::","call_ids":["senseval2.d001.s094.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"most%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.003645,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.440340792061761}
{"item_id":"senseval2.d001.s094.t005","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%3:00:00::","synset_id":"important.a.01"},{"index":2,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"},{"index":3,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":4,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"},{"index":5,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"important%3:00:00::","call_ids":["senseval2.d001.s094.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"important%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01853,"input_uncached_usd":0.00269,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.119217542000115}
{"item_id":"senseval2.d001.s094.t006","gold_sense_keys":["step%1:04:02::"],"candidates":[{"index":1,"sense_key":"step%1:23:01::","synset_id":"step.n.06"},{"index":2,"sense_key":"step%1:04:00::","synset_id":"step.n.03"},{"index":3,"sense_key":"step%1:11:00::","synset_id":"footfall.n.01"},{"index":4,"sense_key":"step%1:10:00::","synset_id":"tone.n.09"},{"index":5,"sense_key":"step%1:04:02::","synset_id":"measure.n.01"},{"index":6,"sense_key":"step%1:23:00::","synset_id":"footstep.n.03"},{"index":7,"sense_key":"step%1:10:02::","synset_id":"footprint.n.01"},{"index":8,"sense_key":"step%1:06:00::","synset_id":"step.n.04"},{"index":9,"sense_key":"step%1:26:00::","synset_id":"gradation.n.01"},{"index":10,"sense_key":"step%1:04:03::","synset_id":"dance_step.n.01"},{"index":11,"sense_key":"step%1:06:01::","synset_id":"step.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"step%1:26:00::","call_ids":["senseval2.d001.s094.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"step%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":779,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019735,"input_uncached_usd":0.0038950000000000005,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.397354791988619}
{"item_id":"senseval2.d001.s094.t007","gold_sense_keys":["genesis%1:11:00::"],"candidates":[{"index":1,"sense_key":"genesis%1:10:00::","synset_id":"genesis.n.02"},{"index":2,"sense_key":"genesis%1:11:00::","synset_id":"genesis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genesis%1:11:00::","call_ids":["senseval2.d001.s094.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genesis%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":411,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.003465,"input_uncached_usd":0.002055,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.304767084075138}
{"item_id":"senseval2.d001.s094.t008","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":2,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":3,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":4,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s094.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":42,"reasoning_output_tokens":26},"cost":{"currency":"USD","total_usd":0.00383,"input_uncached_usd":0.0025700000000000002,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.517506291042082}
{"item_id":"senseval2.d002.s000.t000","gold_sense_keys":["teach%2:32:00::"],"candidates":[{"index":1,"sense_key":"teach%2:32:00::","synset_id":"teach.v.01"},{"index":2,"sense_key":"teach%2:30:00::","synset_id":"teach.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teach%2:32:00::","call_ids":["senseval2.d002.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teach%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":256,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.00425,"input_uncached_usd":0.00128,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2456994170788676}
{"item_id":"senseval2.d002.s000.t001","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":2,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":3,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"child%1:18:00::","call_ids":["senseval2.d002.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"child%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":350,"cached_input_tokens":0,"output_tokens":169,"reasoning_output_tokens":154},"cost":{"currency":"USD","total_usd":0.00682,"input_uncached_usd":0.00175,"input_cached_usd":0.0,"output_usd":0.00507,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.184655958088115}
{"item_id":"senseval2.d002.s000.t002","gold_sense_keys":["read%2:31:00::"],"candidates":[{"index":1,"sense_key":"read%2:36:05::","synset_id":"read.v.09"},{"index":2,"sense_key":"read%2:31:02::","synset_id":"read.v.05"},{"index":3,"sense_key":"read%2:31:01::","synset_id":"read.v.03"},{"index":4,"sense_key":"read%2:31:00::","synset_id":"read.v.01"},{"index":5,"sense_key":"read%2:31:08::","synset_id":"read.v.10"},{"index":6,"sense_key":"read%2:31:06::","synset_id":"read.v.04"},{"index":7,"sense_key":"read%2:31:09::","synset_id":"take.v.06"},{"index":8,"sense_key":"read%2:42:00::","synset_id":"read.v.02"},{"index":9,"sense_key":"read%2:31:05::","synset_id":"learn.v.04"},{"index":10,"sense_key":"read%2:32:02::","synset_id":"read.v.08"},{"index":11,"sense_key":"read%2:31:04::","synset_id":"understand.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"read%2:31:00::","call_ids":["senseval2.d002.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"read%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":655,"cached_input_tokens":0,"output_tokens":143,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.0075650000000000005,"input_uncached_usd":0.003275,"input_cached_usd":0.0,"output_usd":0.00429,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7234595000045374}
{"item_id":"senseval2.d002.s000.t003","gold_sense_keys":["write%2:36:01::"],"candidates":[{"index":1,"sense_key":"write%2:36:03::","synset_id":"spell.v.03"},{"index":2,"sense_key":"write%2:32:00::","synset_id":"write.v.02"},{"index":3,"sense_key":"write%2:36:00::","synset_id":"write.v.01"},{"index":4,"sense_key":"write%2:32:03::","synset_id":"write.v.08"},{"index":5,"sense_key":"write%2:36:10::","synset_id":"write.v.10"},{"index":6,"sense_key":"write%2:36:04::","synset_id":"compose.v.02"},{"index":7,"sense_key":"write%2:36:02::","synset_id":"publish.v.03"},{"index":8,"sense_key":"write%2:32:08::","synset_id":"write.v.04"},{"index":9,"sense_key":"write%2:36:01::","synset_id":"write.v.07"},{"index":10,"sense_key":"write%2:32:01::","synset_id":"write.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"write%2:32:00::","call_ids":["senseval2.d002.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"write%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018735,"input_uncached_usd":0.0028950000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.665855208062567}
{"item_id":"senseval2.d002.s000.t004","gold_sense_keys":["reckon%2:31:01::"],"candidates":[{"index":1,"sense_key":"reckon%2:31:03::","synset_id":"calculate.v.02"},{"index":2,"sense_key":"reckon%2:31:01::","synset_id":"calculate.v.01"},{"index":3,"sense_key":"reckon%2:31:02::","synset_id":"think.v.02"},{"index":4,"sense_key":"reckon%2:31:05::","synset_id":"count.v.08"},{"index":5,"sense_key":"reckon%2:31:13::","synset_id":"reckon.v.06"},{"index":6,"sense_key":"reckon%2:31:00::","synset_id":"see.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"reckon%2:31:01::","call_ids":["senseval2.d002.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"reckon%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.00504,"input_uncached_usd":0.0022800000000000003,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9456042080419138}
{"item_id":"senseval2.d002.s001.t000","gold_sense_keys":["know%2:31:03::"],"candidates":[{"index":1,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":2,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":3,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":4,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":5,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":6,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":7,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":8,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":9,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":10,"sense_key":"know%2:31:14::","synset_id":"know.v.11"},{"index":11,"sense_key":"know%2:31:11::","synset_id":"know.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"know%2:31:03::","call_ids":["senseval2.d002.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"know%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":107},"cost":{"currency":"USD","total_usd":0.007155,"input_uncached_usd":0.0034950000000000003,"input_cached_usd":0.0,"output_usd":0.00366,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.964862791937776}
{"item_id":"senseval2.d002.s002.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":2,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":3,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":4,"sense_key":"want%2:42:01::","synset_id":"want.v.05"},{"index":5,"sense_key":"want%2:35:00::","synset_id":"want.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"want%2:37:00::","call_ids":["senseval2.d002.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.0051400000000000005,"input_uncached_usd":0.00214,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4099042079178616}
{"item_id":"senseval2.d002.s003.t000","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::","reason%1:16:00::"],"candidates":[{"index":1,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":2,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":3,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":4,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":5,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"},{"index":6,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"reason%1:10:01::","call_ids":["senseval2.d002.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"reason%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":542,"cached_input_tokens":0,"output_tokens":180,"reasoning_output_tokens":164},"cost":{"currency":"USD","total_usd":0.008110000000000001,"input_uncached_usd":0.00271,"input_cached_usd":0.0,"output_usd":0.0054,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5341715830145404}
{"item_id":"senseval2.d002.s003.t001","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":2,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":3,"sense_key":"want%2:42:01::","synset_id":"want.v.05"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"want%2:37:00::","call_ids":["senseval2.d002.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":458,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.0052,"input_uncached_usd":0.0022900000000000004,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.173831208026968}
{"item_id":"senseval2.d002.s003.t002","gold_sense_keys":["effective%3:00:00::","effective%5:00:00:competent:00"],"candidates":[{"index":1,"sense_key":"effective%5:00:00:competent:00","synset_id":"effective.s.01"},{"index":2,"sense_key":"effective%5:00:00:operational:00","synset_id":"effective.s.05"},{"index":3,"sense_key":"effective%5:00:00:efficacious:00","synset_id":"effective.s.02"},{"index":4,"sense_key":"effective%5:00:00:actual:00","synset_id":"effective.s.04"},{"index":5,"sense_key":"effective%3:00:00::","synset_id":"effective.a.01"},{"index":6,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"effective%3:00:00::","call_ids":["senseval2.d002.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"effective%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":178,"reasoning_output_tokens":162},"cost":{"currency":"USD","total_usd":0.00825,"input_uncached_usd":0.0029100000000000003,"input_cached_usd":0.0,"output_usd":0.00534,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.350007041008212}
{"item_id":"senseval2.d002.s003.t003","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"},{"index":4,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":5,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":6,"sense_key":"education%1:04:00::","synset_id":"education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":120},"cost":{"currency":"USD","total_usd":0.00672,"input_uncached_usd":0.0026400000000000004,"input_cached_usd":0.0,"output_usd":0.00408,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9016904999734834}
{"item_id":"senseval2.d002.s003.t004","gold_sense_keys":["require%2:42:00::"],"candidates":[{"index":1,"sense_key":"require%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"require%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"require%2:32:01::","synset_id":"ask.v.04"},{"index":4,"sense_key":"require%2:32:00::","synset_id":"command.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"require%2:42:00::","call_ids":["senseval2.d002.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"require%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.0048649999999999995,"input_uncached_usd":0.002135,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.319315791944973}
{"item_id":"senseval2.d002.s003.t005","gold_sense_keys":["relinquish%2:41:00::"],"candidates":[{"index":1,"sense_key":"relinquish%2:35:02::","synset_id":"let_go_of.v.01"},{"index":2,"sense_key":"relinquish%2:40:01::","synset_id":"release.v.08"},{"index":3,"sense_key":"relinquish%2:40:00::","synset_id":"foreswear.v.02"},{"index":4,"sense_key":"relinquish%2:41:00::","synset_id":"waive.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"relinquish%2:41:00::","call_ids":["senseval2.d002.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"relinquish%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":108},"cost":{"currency":"USD","total_usd":0.006165,"input_uncached_usd":0.002445,"input_cached_usd":0.0,"output_usd":0.00372,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4127903329208493}
{"item_id":"senseval2.d002.s003.t008","gold_sense_keys":["belief%1:09:00::"],"candidates":[{"index":1,"sense_key":"belief%1:09:00::","synset_id":"belief.n.01"},{"index":2,"sense_key":"belief%1:09:01::","synset_id":"impression.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"belief%1:09:00::","call_ids":["senseval2.d002.s003.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"belief%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":338,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.00391,"input_uncached_usd":0.00169,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.558643791940995}
{"item_id":"senseval2.d002.s003.t013","gold_sense_keys":["young%3:00:00::"],"candidates":[{"index":1,"sense_key":"young%5:00:00:inexperienced:00","synset_id":"unseasoned.s.02"},{"index":2,"sense_key":"young%5:00:00:new:00","synset_id":"young.s.03"},{"index":3,"sense_key":"young%5:00:01:young:00","synset_id":"youthful.s.01"},{"index":4,"sense_key":"young%3:00:00::","synset_id":"young.a.01"},{"index":5,"sense_key":"young%5:00:00:early:02","synset_id":"new.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"young%3:00:00::","call_ids":["senseval2.d002.s003.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"young%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004395,"input_uncached_usd":0.002415,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2765956670045853}
{"item_id":"senseval2.d002.s003.t014","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"},{"index":3,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":4,"sense_key":"people%1:14:02::","synset_id":"people.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval2.d002.s003.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.004835000000000001,"input_uncached_usd":0.0021950000000000003,"input_cached_usd":0.0,"output_usd":0.00264,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4753034170717}
{"item_id":"senseval2.d002.s003.t016","gold_sense_keys":["violate%2:41:00::"],"candidates":[{"index":1,"sense_key":"violate%2:35:00::","synset_id":"violate.v.03"},{"index":2,"sense_key":"violate%2:42:00::","synset_id":"violate.v.01"},{"index":3,"sense_key":"violate%2:41:00::","synset_id":"transgress.v.01"},{"index":4,"sense_key":"violate%2:41:01::","synset_id":"rape.v.01"},{"index":5,"sense_key":"violate%2:41:02::","synset_id":"desecrate.v.01"},{"index":6,"sense_key":"violate%2:35:01::","synset_id":"rape.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"violate%2:35:00::","call_ids":["senseval2.d002.s003.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"violate%2:35:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":1564,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.049735,"input_uncached_usd":0.0028150000000000002,"input_cached_usd":0.0,"output_usd":0.04692,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":24.545952791930176}
{"item_id":"senseval2.d002.s004.t000","gold_sense_keys":["belief%1:09:00::"],"candidates":[{"index":1,"sense_key":"belief%1:09:00::","synset_id":"belief.n.01"},{"index":2,"sense_key":"belief%1:09:01::","synset_id":"impression.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"belief%1:09:00::","call_ids":["senseval2.d002.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"belief%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":343,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.004295,"input_uncached_usd":0.0017150000000000002,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9390258330386132}
{"item_id":"senseval2.d002.s004.t001","gold_sense_keys":["so%4:02:03::"],"candidates":[{"index":1,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":2,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"},{"index":3,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":4,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":5,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":6,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":7,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":8,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":9,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":10,"sense_key":"so%4:02:02::","synset_id":"so.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"so%4:02:03::","call_ids":["senseval2.d002.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"so%4:02:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.006265000000000001,"input_uncached_usd":0.0037450000000000005,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.078409833018668}
{"item_id":"senseval2.d002.s004.t003","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":330,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.00336,"input_uncached_usd":0.0016500000000000002,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8565635409904644}
{"item_id":"senseval2.d002.s004.t004","gold_sense_keys":["establishment%1:14:00::","establishment%1:14:02::"],"candidates":[{"index":1,"sense_key":"establishment%1:09:00::","synset_id":"establishment.n.07"},{"index":2,"sense_key":"establishment%1:04:00::","synset_id":"constitution.n.02"},{"index":3,"sense_key":"establishment%1:14:02::","synset_id":"administration.n.02"},{"index":4,"sense_key":"establishment%1:14:01::","synset_id":"establishment.n.05"},{"index":5,"sense_key":"establishment%1:14:00::","synset_id":"institution.n.01"},{"index":6,"sense_key":"establishment%1:06:00::","synset_id":"establishment.n.04"},{"index":7,"sense_key":"establishment%1:22:00::","synset_id":"establishment.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"establishment%1:14:02::","call_ids":["senseval2.d002.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"establishment%1:14:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":2082,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.065235,"input_uncached_usd":0.002775,"input_cached_usd":0.0,"output_usd":0.06246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":33.40726245800033}
{"item_id":"senseval2.d002.s004.t005","gold_sense_keys":["politician%1:18:00::","politician%1:18:01::"],"candidates":[{"index":1,"sense_key":"politician%1:18:01::","synset_id":"politician.n.01"},{"index":2,"sense_key":"politician%1:18:02::","synset_id":"politician.n.03"},{"index":3,"sense_key":"politician%1:18:00::","synset_id":"politician.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"politician%1:18:00::","call_ids":["senseval2.d002.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"politician%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":357,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.0049949999999999994,"input_uncached_usd":0.0017850000000000001,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5732924999902025}
{"item_id":"senseval2.d002.s004.t006","gold_sense_keys":["even%4:02:00::"],"candidates":[{"index":1,"sense_key":"even%4:02:00::","synset_id":"even.r.01"},{"index":2,"sense_key":"even%4:02:04::","synset_id":"even.r.04"},{"index":3,"sense_key":"even%4:02:03::","synset_id":"even.r.02"},{"index":4,"sense_key":"even%4:02:01::","synset_id":"even.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"even%4:02:00::","call_ids":["senseval2.d002.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"even%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.00417,"input_uncached_usd":0.00216,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.99131683295127}
{"item_id":"senseval2.d002.s004.t007","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"},{"index":2,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":343,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.003575,"input_uncached_usd":0.0017150000000000002,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.880389375030063}
{"item_id":"senseval2.d002.s004.t008","gold_sense_keys":["seem%2:39:01::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"seem%2:39:01::","call_ids":["senseval2.d002.s004.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"seem%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018090000000000002,"input_uncached_usd":0.0022500000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.027013125014491}
{"item_id":"senseval2.d002.s004.t010","gold_sense_keys":["blasphemous%5:00:00:irreverent:00"],"candidates":[{"index":1,"sense_key":"blasphemous%5:00:00:irreverent:00","synset_id":"blasphemous.s.01"},{"index":2,"sense_key":"blasphemous%5:00:00:dirty:02","synset_id":"blasphemous.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"blasphemous%5:00:00:irreverent:00","call_ids":["senseval2.d002.s004.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"blasphemous%5:00:00:irreverent:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":391,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.004145,"input_uncached_usd":0.001955,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7836451249895617}
{"item_id":"senseval2.d002.s004.t011","gold_sense_keys":["challenge%2:32:01::"],"candidates":[{"index":1,"sense_key":"challenge%2:32:03::","synset_id":"challenge.v.03"},{"index":2,"sense_key":"challenge%2:32:00::","synset_id":"challenge.v.02"},{"index":3,"sense_key":"challenge%2:32:02::","synset_id":"challenge.v.04"},{"index":4,"sense_key":"challenge%2:32:01::","synset_id":"challenge.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"challenge%2:32:01::","call_ids":["senseval2.d002.s004.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"challenge%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":397,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.004175,"input_uncached_usd":0.0019850000000000002,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.441701582982205}
{"item_id":"senseval2.d002.s005.t000","gold_sense_keys":["here%4:02:02::"],"candidates":[{"index":1,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":2,"sense_key":"here%4:02:04::","synset_id":"here.r.04"},{"index":3,"sense_key":"here%4:02:00::","synset_id":"here.r.01"},{"index":4,"sense_key":"here%4:02:02::","synset_id":"here.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"here%4:02:02::","call_ids":["senseval2.d002.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"here%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01815,"input_uncached_usd":0.00231,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.88887495896779}
{"item_id":"senseval2.d002.s005.t001","gold_sense_keys":["example%1:09:00::"],"candidates":[{"index":1,"sense_key":"example%1:10:00::","synset_id":"example.n.04"},{"index":2,"sense_key":"example%1:11:00::","synset_id":"case.n.01"},{"index":3,"sense_key":"example%1:09:02::","synset_id":"model.n.07"},{"index":4,"sense_key":"example%1:09:01::","synset_id":"exemplar.n.01"},{"index":5,"sense_key":"example%1:04:00::","synset_id":"exercise.n.04"},{"index":6,"sense_key":"example%1:09:00::","synset_id":"example.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"example%1:09:00::","call_ids":["senseval2.d002.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"example%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":80},"cost":{"currency":"USD","total_usd":0.00579,"input_uncached_usd":0.0029100000000000003,"input_cached_usd":0.0,"output_usd":0.00288,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.318364832899533}
{"item_id":"senseval2.d002.s006.t000","gold_sense_keys":["ask%2:32:04::"],"candidates":[{"index":1,"sense_key":"ask%2:32:04::","synset_id":"ask.v.05"},{"index":2,"sense_key":"ask%2:32:09::","synset_id":"ask.v.07"},{"index":3,"sense_key":"ask%2:32:00::","synset_id":"ask.v.01"},{"index":4,"sense_key":"ask%2:32:01::","synset_id":"ask.v.02"},{"index":5,"sense_key":"ask%2:32:02::","synset_id":"ask.v.03"},{"index":6,"sense_key":"ask%2:42:00::","synset_id":"necessitate.v.01"},{"index":7,"sense_key":"ask%2:32:05::","synset_id":"ask.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ask%2:32:04::","call_ids":["senseval2.d002.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ask%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":147},"cost":{"currency":"USD","total_usd":0.00796,"input_uncached_usd":0.0031000000000000003,"input_cached_usd":0.0,"output_usd":0.00486,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3952343750279397}
{"item_id":"senseval2.d002.s006.t001","gold_sense_keys":["sample%1:09:00::","sample%1:09:01::"],"candidates":[{"index":1,"sense_key":"sample%1:09:01::","synset_id":"sample_distribution.n.01"},{"index":2,"sense_key":"sample%1:09:00::","synset_id":"sample.n.01"},{"index":3,"sense_key":"sample%1:17:00::","synset_id":"sample.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sample%1:09:01::","call_ids":["senseval2.d002.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sample%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":458,"reasoning_output_tokens":443},"cost":{"currency":"USD","total_usd":0.015785,"input_uncached_usd":0.0020450000000000004,"input_cached_usd":0.0,"output_usd":0.01374,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.760558666079305}
{"item_id":"senseval2.d002.s006.t002","gold_sense_keys":["american%3:01:01::"],"candidates":[{"index":1,"sense_key":"american%3:01:01::","synset_id":"american.a.01"},{"index":2,"sense_key":"american%3:01:00::","synset_id":"american.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%3:01:01::","call_ids":["senseval2.d002.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":391,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.004565,"input_uncached_usd":0.001955,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.873905999935232}
{"item_id":"senseval2.d002.s006.t003","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"},{"index":2,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":384,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.00354,"input_uncached_usd":0.0019200000000000003,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1104574169730768}
{"item_id":"senseval2.d002.s006.t006","gold_sense_keys":["encourage%2:41:00::"],"candidates":[{"index":1,"sense_key":"encourage%2:37:00::","synset_id":"encourage.v.02"},{"index":2,"sense_key":"encourage%2:41:00::","synset_id":"promote.v.01"},{"index":3,"sense_key":"encourage%2:32:00::","synset_id":"encourage.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"encourage%2:41:00::","call_ids":["senseval2.d002.s006.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"encourage%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.004135,"input_uncached_usd":0.002095,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8352651249151677}
{"item_id":"senseval2.d002.s006.t008","gold_sense_keys":["child%1:18:01::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d002.s006.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"child%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":393,"reasoning_output_tokens":377},"cost":{"currency":"USD","total_usd":0.014195,"input_uncached_usd":0.002405,"input_cached_usd":0.0,"output_usd":0.01179,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.387147541041486}
{"item_id":"senseval2.d002.s006.t009","gold_sense_keys":["answer%1:10:01::"],"candidates":[{"index":1,"sense_key":"answer%1:04:00::","synset_id":"answer.n.05"},{"index":2,"sense_key":"answer%1:10:03::","synset_id":"answer.n.04"},{"index":3,"sense_key":"answer%1:10:02::","synset_id":"solution.n.02"},{"index":4,"sense_key":"answer%1:10:00::","synset_id":"answer.n.03"},{"index":5,"sense_key":"answer%1:10:01::","synset_id":"answer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"answer%1:10:01::","call_ids":["senseval2.d002.s006.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"answer%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":112},"cost":{"currency":"USD","total_usd":0.00677,"input_uncached_usd":0.0029300000000000003,"input_cached_usd":0.0,"output_usd":0.00384,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3064567500259727}
{"item_id":"senseval2.d002.s007.t000","gold_sense_keys":["mean%2:32:01::"],"candidates":[{"index":1,"sense_key":"mean%2:31:00::","synset_id":"intend.v.01"},{"index":2,"sense_key":"mean%2:32:03::","synset_id":"mean.v.03"},{"index":3,"sense_key":"mean%2:42:03::","synset_id":"mean.v.05"},{"index":4,"sense_key":"mean%2:32:01::","synset_id":"mean.v.01"},{"index":5,"sense_key":"mean%2:31:05::","synset_id":"mean.v.07"},{"index":6,"sense_key":"mean%2:42:00::","synset_id":"entail.v.01"},{"index":7,"sense_key":"mean%2:31:01::","synset_id":"think_of.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"mean%2:32:01::","call_ids":["senseval2.d002.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"mean%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018885,"input_uncached_usd":0.0030450000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.820034125004895}
{"item_id":"senseval2.d002.s008.t000","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval2.d002.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.0048000000000000004,"input_uncached_usd":0.0020700000000000002,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0051263329805806}
{"item_id":"senseval2.d002.s009.t002","gold_sense_keys":["equate%2:31:00::"],"candidates":[{"index":1,"sense_key":"equate%2:30:00::","synset_id":"equal.v.03"},{"index":2,"sense_key":"equate%2:31:00::","synset_id":"compare.v.03"},{"index":3,"sense_key":"equate%2:42:00::","synset_id":"equate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"equate%2:31:00::","call_ids":["senseval2.d002.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"equate%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.00421,"input_uncached_usd":0.00223,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5916111669503152}
{"item_id":"senseval2.d002.s009.t004","gold_sense_keys":["encourage%2:41:00::"],"candidates":[{"index":1,"sense_key":"encourage%2:37:00::","synset_id":"encourage.v.02"},{"index":2,"sense_key":"encourage%2:41:00::","synset_id":"promote.v.01"},{"index":3,"sense_key":"encourage%2:32:00::","synset_id":"encourage.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"encourage%2:41:00::","call_ids":["senseval2.d002.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"encourage%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":169,"reasoning_output_tokens":153},"cost":{"currency":"USD","total_usd":0.007090000000000001,"input_uncached_usd":0.00202,"input_cached_usd":0.0,"output_usd":0.00507,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.719727790914476}
{"item_id":"senseval2.d002.s010.t000","gold_sense_keys":["result%1:19:00::"],"candidates":[{"index":1,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":2,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":3,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"},{"index":4,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"result%1:19:00::","call_ids":["senseval2.d002.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"result%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":539,"cached_input_tokens":0,"output_tokens":699,"reasoning_output_tokens":678},"cost":{"currency":"USD","total_usd":0.023665,"input_uncached_usd":0.0026950000000000003,"input_cached_usd":0.0,"output_usd":0.02097,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":20.110604416928254}
{"item_id":"senseval2.d002.s010.t001","gold_sense_keys":["generation%1:14:01::"],"candidates":[{"index":1,"sense_key":"generation%1:28:00::","synset_id":"generation.n.03"},{"index":2,"sense_key":"generation%1:04:00::","synset_id":"generation.n.07"},{"index":3,"sense_key":"generation%1:14:01::","synset_id":"coevals.n.01"},{"index":4,"sense_key":"generation%1:04:01::","synset_id":"generation.n.06"},{"index":5,"sense_key":"generation%1:28:01::","synset_id":"generation.n.04"},{"index":6,"sense_key":"generation%1:14:00::","synset_id":"generation.n.02"},{"index":7,"sense_key":"generation%1:11:00::","synset_id":"genesis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"generation%1:14:01::","call_ids":["senseval2.d002.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"generation%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":562,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.00575,"input_uncached_usd":0.0028100000000000004,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2193187089869753}
{"item_id":"senseval2.d002.s010.t002","gold_sense_keys":["young%3:00:00::"],"candidates":[{"index":1,"sense_key":"young%5:00:01:young:00","synset_id":"youthful.s.01"},{"index":2,"sense_key":"young%5:00:00:new:00","synset_id":"young.s.03"},{"index":3,"sense_key":"young%3:00:00::","synset_id":"young.a.01"},{"index":4,"sense_key":"young%5:00:00:inexperienced:00","synset_id":"unseasoned.s.02"},{"index":5,"sense_key":"young%5:00:00:early:02","synset_id":"new.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"young%3:00:00::","call_ids":["senseval2.d002.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"young%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.004225,"input_uncached_usd":0.002755,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.858820499968715}
{"item_id":"senseval2.d002.s010.t003","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":2,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":3,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval2.d002.s010.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":507,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.004155000000000001,"input_uncached_usd":0.0025350000000000004,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7866233750246465}
{"item_id":"senseval2.d002.s010.t006","gold_sense_keys":["incompetence%1:07:00::"],"candidates":[{"index":1,"sense_key":"incompetence%1:07:00::","synset_id":"incompetence.n.01"},{"index":2,"sense_key":"incompetence%1:26:00::","synset_id":"incompetence.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"incompetence%1:07:00::","call_ids":["senseval2.d002.s010.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"incompetence%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.0035700000000000003,"input_uncached_usd":0.0019500000000000001,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3479297909652814}
{"item_id":"senseval2.d002.s010.t007","gold_sense_keys":["match%2:42:01::"],"candidates":[{"index":1,"sense_key":"match%2:40:00::","synset_id":"match.v.02"},{"index":2,"sense_key":"match%2:42:00::","synset_id":"match.v.01"},{"index":3,"sense_key":"match%2:42:01::","synset_id":"equal.v.02"},{"index":4,"sense_key":"match%2:30:01::","synset_id":"match.v.05"},{"index":5,"sense_key":"match%2:35:10::","synset_id":"match.v.07"},{"index":6,"sense_key":"match%2:41:00::","synset_id":"meet.v.06"},{"index":7,"sense_key":"match%2:35:08::","synset_id":"match.v.03"},{"index":8,"sense_key":"match%2:30:07::","synset_id":"equal.v.03"},{"index":9,"sense_key":"match%2:30:00::","synset_id":"match.v.09"},{"index":10,"sense_key":"match%2:33:00::","synset_id":"pit.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"match%2:42:01::","call_ids":["senseval2.d002.s010.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"match%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":775,"cached_input_tokens":0,"output_tokens":193,"reasoning_output_tokens":178},"cost":{"currency":"USD","total_usd":0.009665,"input_uncached_usd":0.0038750000000000004,"input_cached_usd":0.0,"output_usd":0.00579,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.496797125088051}
{"item_id":"senseval2.d002.s010.t008","gold_sense_keys":["only%4:02:01::"],"candidates":[{"index":1,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":2,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":3,"sense_key":"only%4:02:05::","synset_id":"only.r.07"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":6,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":7,"sense_key":"only%4:02:00::","synset_id":"only.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"only%4:02:01::","call_ids":["senseval2.d002.s010.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"only%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":220,"reasoning_output_tokens":204},"cost":{"currency":"USD","total_usd":0.009645,"input_uncached_usd":0.0030450000000000004,"input_cached_usd":0.0,"output_usd":0.0066,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.398544332943857}
{"item_id":"senseval2.d002.s010.t009","gold_sense_keys":["good%3:00:01::"],"candidates":[{"index":1,"sense_key":"good%5:00:00:discriminating:00","synset_id":"good.s.19"},{"index":2,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":3,"sense_key":"good%5:00:00:healthful:00","synset_id":"good.s.16"},{"index":4,"sense_key":"good%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":5,"sense_key":"good%5:00:00:fresh:01","synset_id":"good.s.18"},{"index":6,"sense_key":"good%5:00:00:genuine:00","synset_id":"good.s.17"},{"index":7,"sense_key":"good%5:00:00:reputable:00","synset_id":"estimable.s.01"},{"index":8,"sense_key":"good%5:00:00:advantageous:00","synset_id":"beneficial.s.01"},{"index":9,"sense_key":"good%5:00:00:nice:00","synset_id":"good.s.04"},{"index":10,"sense_key":"good%5:00:00:pleasing:00","synset_id":"good.s.13"},{"index":11,"sense_key":"good%5:00:00:complete:00","synset_id":"good.s.07"},{"index":12,"sense_key":"good%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":13,"sense_key":"good%5:00:00:skilled:00","synset_id":"adept.s.01"},{"index":14,"sense_key":"good%3:00:02::","synset_id":"good.a.03"},{"index":15,"sense_key":"good%5:00:00:close:02","synset_id":"dear.s.02"},{"index":16,"sense_key":"good%5:00:00:opportune:00","synset_id":"good.s.10"},{"index":17,"sense_key":"good%5:00:00:ample:00","synset_id":"full.s.04"},{"index":18,"sense_key":"good%5:00:01:sound:00","synset_id":"dependable.s.03"},{"index":19,"sense_key":"good%5:00:00:healthy:00","synset_id":"good.s.15"},{"index":20,"sense_key":"good%3:00:01::","synset_id":"good.a.01"},{"index":21,"sense_key":"good%5:00:00:righteous:00","synset_id":"good.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":20,"chosen_sense_key":"good%3:00:01::","call_ids":["senseval2.d002.s010.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":20,"predicted_sense_key":"good%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1209,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.021885,"input_uncached_usd":0.006045000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.060407417011447}
{"item_id":"senseval2.d002.s010.t010","gold_sense_keys":["opinion%1:09:00::"],"candidates":[{"index":1,"sense_key":"opinion%1:09:00::","synset_id":"opinion.n.01"},{"index":2,"sense_key":"opinion%1:09:01::","synset_id":"public_opinion.n.01"},{"index":3,"sense_key":"opinion%1:04:01::","synset_id":"opinion.n.05"},{"index":4,"sense_key":"opinion%1:09:02::","synset_id":"impression.n.01"},{"index":5,"sense_key":"opinion%1:10:01::","synset_id":"opinion.n.02"},{"index":6,"sense_key":"opinion%1:10:00::","synset_id":"opinion.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"opinion%1:09:00::","call_ids":["senseval2.d002.s010.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"opinion%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":160,"reasoning_output_tokens":144},"cost":{"currency":"USD","total_usd":0.007985,"input_uncached_usd":0.0031850000000000003,"input_cached_usd":0.0,"output_usd":0.0048000000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.993809042032808}
{"item_id":"senseval2.d002.s011.t000","gold_sense_keys":["whole%3:00:00::"],"candidates":[{"index":1,"sense_key":"whole%5:00:00:healthy:00","synset_id":"hale.s.01"},{"index":2,"sense_key":"whole%3:00:00::","synset_id":"whole.a.01"},{"index":3,"sense_key":"whole%5:00:00:undiversified:00","synset_id":"solid.s.13"},{"index":4,"sense_key":"whole%5:00:00:uninjured:00","synset_id":"unharmed.s.01"},{"index":5,"sense_key":"whole%3:00:02::","synset_id":"whole.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"whole%3:00:00::","call_ids":["senseval2.d002.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"whole%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.004895,"input_uncached_usd":0.002735,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9354434580309317}
{"item_id":"senseval2.d002.s011.t001","gold_sense_keys":["notion%1:09:00::"],"candidates":[{"index":1,"sense_key":"notion%1:09:01::","synset_id":"notion.n.03"},{"index":2,"sense_key":"notion%1:06:00::","synset_id":"notion.n.04"},{"index":3,"sense_key":"notion%1:09:00::","synset_id":"notion.n.02"},{"index":4,"sense_key":"notion%1:09:02::","synset_id":"impression.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"notion%1:09:00::","call_ids":["senseval2.d002.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"notion%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":502,"reasoning_output_tokens":487},"cost":{"currency":"USD","total_usd":0.01764,"input_uncached_usd":0.0025800000000000003,"input_cached_usd":0.0,"output_usd":0.01506,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.544574916013516}
{"item_id":"senseval2.d002.s011.t003","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":2,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":3,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":4,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"},{"index":5,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":6,"sense_key":"education%1:04:01::","synset_id":"education.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":109},"cost":{"currency":"USD","total_usd":0.006775,"input_uncached_usd":0.0030250000000000003,"input_cached_usd":0.0,"output_usd":0.0037500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2427523341029882}
{"item_id":"senseval2.d002.s011.t004","gold_sense_keys":["romantic%5:00:00:impractical:00"],"candidates":[{"index":1,"sense_key":"romantic%5:00:01:loving:00","synset_id":"amatory.s.01"},{"index":2,"sense_key":"romantic%3:01:00::","synset_id":"romantic.a.01"},{"index":3,"sense_key":"romantic%5:00:00:impractical:00","synset_id":"quixotic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"romantic%5:00:00:impractical:00","call_ids":["senseval2.d002.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"romantic%5:00:00:impractical:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018345,"input_uncached_usd":0.0025050000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.774706084048375}
{"item_id":"senseval2.d002.s011.t005","gold_sense_keys":["rebellion%1:04:01::"],"candidates":[{"index":1,"sense_key":"rebellion%1:04:01::","synset_id":"rebellion.n.01"},{"index":2,"sense_key":"rebellion%1:04:00::","synset_id":"rebellion.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rebellion%1:04:01::","call_ids":["senseval2.d002.s011.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rebellion%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.00442,"input_uncached_usd":0.00217,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2255693341139704}
{"item_id":"senseval2.d002.s011.t006","gold_sense_keys":["disciplined%5:00:00:controlled:00"],"candidates":[{"index":1,"sense_key":"disciplined%5:00:00:controlled:00","synset_id":"disciplined.s.01"},{"index":2,"sense_key":"disciplined%5:00:00:trained:00","synset_id":"disciplined.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"disciplined%5:00:00:controlled:00","call_ids":["senseval2.d002.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"disciplined%5:00:00:controlled:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":411,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017865000000000002,"input_uncached_usd":0.002055,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.404332250007428}
{"item_id":"senseval2.d002.s011.t007","gold_sense_keys":["instruction%1:04:01::"],"candidates":[{"index":1,"sense_key":"instruction%1:04:01::","synset_id":"education.n.01"},{"index":2,"sense_key":"instruction%1:10:02::","synset_id":"instruction.n.04"},{"index":3,"sense_key":"instruction%1:10:04::","synset_id":"direction.n.06"},{"index":4,"sense_key":"instruction%1:04:00::","synset_id":"teaching.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"instruction%1:04:01::","call_ids":["senseval2.d002.s011.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"instruction%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00456,"input_uncached_usd":0.0026400000000000004,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6148067499743775}
{"item_id":"senseval2.d002.s011.t009","gold_sense_keys":["authoritarian%5:00:00:domineering:00"],"candidates":[{"index":1,"sense_key":"authoritarian%5:00:00:undemocratic:00","synset_id":"authoritarian.s.01"},{"index":2,"sense_key":"authoritarian%5:00:00:domineering:00","synset_id":"authoritarian.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"authoritarian%5:00:00:domineering:00","call_ids":["senseval2.d002.s011.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"authoritarian%5:00:00:domineering:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":88},"cost":{"currency":"USD","total_usd":0.005365,"input_uncached_usd":0.002275,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.83175062504597}
{"item_id":"senseval2.d002.s011.t010","gold_sense_keys":["repression%1:04:00::"],"candidates":[{"index":1,"sense_key":"repression%1:22:00::","synset_id":"repression.n.02"},{"index":2,"sense_key":"repression%1:04:00::","synset_id":"repression.n.03"},{"index":3,"sense_key":"repression%1:26:00::","synset_id":"repression.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"repression%1:04:00::","call_ids":["senseval2.d002.s011.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"repression%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":73},"cost":{"currency":"USD","total_usd":0.00494,"input_uncached_usd":0.0022700000000000003,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2742160839261487}
{"item_id":"senseval2.d002.s011.t011","gold_sense_keys":["frustration%1:04:02::"],"candidates":[{"index":1,"sense_key":"frustration%1:12:00::","synset_id":"frustration.n.01"},{"index":2,"sense_key":"frustration%1:12:01::","synset_id":"frustration.n.03"},{"index":3,"sense_key":"frustration%1:04:02::","synset_id":"frustration.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"frustration%1:04:02::","call_ids":["senseval2.d002.s011.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"frustration%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":73},"cost":{"currency":"USD","total_usd":0.00495,"input_uncached_usd":0.0022800000000000003,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2727656250353903}
{"item_id":"senseval2.d002.s011.t012","gold_sense_keys":["latent%5:00:00:potential:00"],"candidates":[{"index":1,"sense_key":"latent%5:00:00:inactive:02","synset_id":"latent.s.02"},{"index":2,"sense_key":"latent%5:00:00:potential:00","synset_id":"latent.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"latent%5:00:00:potential:00","call_ids":["senseval2.d002.s011.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"latent%5:00:00:potential:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.003935,"input_uncached_usd":0.0020450000000000004,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7903199159773067}
{"item_id":"senseval2.d002.s011.t013","gold_sense_keys":["talent%1:09:00::"],"candidates":[{"index":1,"sense_key":"talent%1:18:00::","synset_id":"talent.n.02"},{"index":2,"sense_key":"talent%1:09:00::","synset_id":"endowment.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"talent%1:09:00::","call_ids":["senseval2.d002.s011.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"talent%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.0038,"input_uncached_usd":0.002,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6204829170601442}
{"item_id":"senseval2.d002.s011.t016","gold_sense_keys":["yet%4:02:02::"],"candidates":[{"index":1,"sense_key":"yet%4:02:06::","synset_id":"however.r.01"},{"index":2,"sense_key":"yet%4:02:05::","synset_id":"so_far.r.03"},{"index":3,"sense_key":"yet%4:02:04::","synset_id":"so_far.r.01"},{"index":4,"sense_key":"yet%4:02:03::","synset_id":"even.r.03"},{"index":5,"sense_key":"yet%4:02:02::","synset_id":"yet.r.01"},{"index":6,"sense_key":"yet%4:02:00::","synset_id":"yet.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"yet%4:02:04::","call_ids":["senseval2.d002.s011.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"yet%4:02:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":678,"cached_input_tokens":0,"output_tokens":176,"reasoning_output_tokens":160},"cost":{"currency":"USD","total_usd":0.00867,"input_uncached_usd":0.0033900000000000002,"input_cached_usd":0.0,"output_usd":0.00528,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.907122667063959}
{"item_id":"senseval2.d002.s011.t018","gold_sense_keys":["potentiality%1:09:00::"],"candidates":[{"index":1,"sense_key":"potentiality%1:09:00::","synset_id":"capability.n.03"},{"index":2,"sense_key":"potentiality%1:26:00::","synset_id":"potential.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"potentiality%1:09:00::","call_ids":["senseval2.d002.s011.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"potentiality%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":278,"reasoning_output_tokens":262},"cost":{"currency":"USD","total_usd":0.010360000000000001,"input_uncached_usd":0.00202,"input_cached_usd":0.0,"output_usd":0.00834,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.433910000021569}
{"item_id":"senseval2.d002.s011.t019","gold_sense_keys":["inherent%5:00:00:intrinsic:00"],"candidates":[{"index":1,"sense_key":"inherent%5:00:00:intrinsic:00","synset_id":"built-in.s.01"},{"index":2,"sense_key":"inherent%5:00:00:implicit:00","synset_id":"implicit_in.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"inherent%5:00:00:implicit:00","call_ids":["senseval2.d002.s011.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"inherent%5:00:00:implicit:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.033645,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":19.492891249945387}
{"item_id":"senseval2.d002.s011.t020","gold_sense_keys":["soul%1:18:00::"],"candidates":[{"index":1,"sense_key":"soul%1:03:00::","synset_id":"person.n.01"},{"index":2,"sense_key":"soul%1:12:00::","synset_id":"soul.n.03"},{"index":3,"sense_key":"soul%1:18:00::","synset_id":"soul.n.01"},{"index":4,"sense_key":"soul%1:04:00::","synset_id":"soul.n.04"},{"index":5,"sense_key":"soul%1:10:00::","synset_id":"soul.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"soul%1:18:00::","call_ids":["senseval2.d002.s011.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"soul%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.0046700000000000005,"input_uncached_usd":0.0026300000000000004,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.767990457941778}
{"item_id":"senseval2.d002.s011.t021","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:07::","synset_id":"child.n.04"},{"index":2,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":3,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":4,"sense_key":"child%1:18:00::","synset_id":"child.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"child%1:18:00::","call_ids":["senseval2.d002.s011.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"child%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":360,"reasoning_output_tokens":344},"cost":{"currency":"USD","total_usd":0.013355,"input_uncached_usd":0.0025550000000000004,"input_cached_usd":0.0,"output_usd":0.0108,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.484554124996066}
{"item_id":"senseval2.d002.s012.t001","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.0037550000000000005,"input_uncached_usd":0.0020150000000000003,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5435511670075357}
{"item_id":"senseval2.d002.s012.t002","gold_sense_keys":["find%2:31:10::"],"candidates":[{"index":1,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":2,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":3,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":4,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":5,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"},{"index":6,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":7,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":8,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":9,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":10,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":11,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":12,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":13,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":14,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":15,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":16,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"find%2:31:10::","call_ids":["senseval2.d002.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"find%2:31:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1118,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.00868,"input_uncached_usd":0.00559,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.466553584090434}
{"item_id":"senseval2.d002.s012.t003","gold_sense_keys":["romantic%5:00:00:impractical:00"],"candidates":[{"index":1,"sense_key":"romantic%3:01:00::","synset_id":"romantic.a.01"},{"index":2,"sense_key":"romantic%5:00:01:loving:00","synset_id":"amatory.s.01"},{"index":3,"sense_key":"romantic%5:00:00:impractical:00","synset_id":"quixotic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"romantic%5:00:00:impractical:00","call_ids":["senseval2.d002.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"romantic%5:00:00:impractical:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":97},"cost":{"currency":"USD","total_usd":0.005835,"input_uncached_usd":0.002445,"input_cached_usd":0.0,"output_usd":0.0033900000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3551965409424156}
{"item_id":"senseval2.d002.s012.t004","gold_sense_keys":["so%4:02:02::"],"candidates":[{"index":1,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"},{"index":2,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":3,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":4,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":5,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":6,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":7,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":8,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":9,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":10,"sense_key":"so%4:02:05::","synset_id":"so.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"so%4:02:02::","call_ids":["senseval2.d002.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"so%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":254,"reasoning_output_tokens":238},"cost":{"currency":"USD","total_usd":0.011665000000000002,"input_uncached_usd":0.004045,"input_cached_usd":0.0,"output_usd":0.00762,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.148675041971728}
{"item_id":"senseval2.d002.s012.t005","gold_sense_keys":["attractive%5:00:00:appealing:00"],"candidates":[{"index":1,"sense_key":"attractive%5:00:00:appealing:00","synset_id":"attractive.s.01"},{"index":2,"sense_key":"attractive%3:00:02::","synset_id":"attractive.a.03"},{"index":3,"sense_key":"attractive%3:00:01::","synset_id":"attractive.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attractive%5:00:00:appealing:00","call_ids":["senseval2.d002.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attractive%5:00:00:appealing:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":493,"reasoning_output_tokens":477},"cost":{"currency":"USD","total_usd":0.017025,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.014790000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.8526721249800175}
{"item_id":"senseval2.d002.s013.t001","gold_sense_keys":["same%3:00:02::"],"candidates":[{"index":1,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":2,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"},{"index":3,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":4,"sense_key":"same%3:00:00::","synset_id":"same.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"same%3:00:02::","call_ids":["senseval2.d002.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"same%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":93},"cost":{"currency":"USD","total_usd":0.00572,"input_uncached_usd":0.0024500000000000004,"input_cached_usd":0.0,"output_usd":0.00327,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.16459462500643}
{"item_id":"senseval2.d002.s013.t002","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"},{"index":2,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":405,"cached_input_tokens":0,"output_tokens":41,"reasoning_output_tokens":26},"cost":{"currency":"USD","total_usd":0.003255,"input_uncached_usd":0.0020250000000000003,"input_cached_usd":0.0,"output_usd":0.00123,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2403052920708433}
{"item_id":"senseval2.d002.s013.t003","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:42:01::","synset_id":"want.v.05"},{"index":2,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":5,"sense_key":"want%2:34:00::","synset_id":"want.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"want%2:37:00::","call_ids":["senseval2.d002.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.0060750000000000005,"input_uncached_usd":0.002625,"input_cached_usd":0.0,"output_usd":0.00345,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.610535416053608}
{"item_id":"senseval2.d002.s013.t004","gold_sense_keys":["child%1:18:01::"],"candidates":[{"index":1,"sense_key":"child%1:18:07::","synset_id":"child.n.04"},{"index":2,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":3,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":4,"sense_key":"child%1:18:06::","synset_id":"child.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d002.s013.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"child%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":147},"cost":{"currency":"USD","total_usd":0.00737,"input_uncached_usd":0.00251,"input_cached_usd":0.0,"output_usd":0.00486,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.203495833906345}
{"item_id":"senseval2.d002.s013.t005","gold_sense_keys":["get%2:40:00::"],"candidates":[{"index":1,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":2,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":3,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":4,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":5,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":6,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":7,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":8,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":9,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":10,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":11,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":12,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":13,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":15,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":16,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":17,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":18,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":19,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":20,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":21,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":22,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":23,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":24,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":25,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":26,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":27,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":28,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":29,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":30,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":31,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":32,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":33,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":34,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"},{"index":35,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":36,"sense_key":"get%2:40:14::","synset_id":"get.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"get%2:40:00::","call_ids":["senseval2.d002.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"get%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1961,"cached_input_tokens":0,"output_tokens":2258,"reasoning_output_tokens":2219},"cost":{"currency":"USD","total_usd":0.077545,"input_uncached_usd":0.009805000000000001,"input_cached_usd":0.0,"output_usd":0.06774000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":34.76984141592402}
{"item_id":"senseval2.d002.s013.t006","gold_sense_keys":["decent%5:00:00:sufficient:00"],"candidates":[{"index":1,"sense_key":"decent%5:00:00:proper:00","synset_id":"becoming.s.01"},{"index":2,"sense_key":"decent%5:00:00:respectable:00","synset_id":"decent.s.01"},{"index":3,"sense_key":"decent%5:00:00:modest:01","synset_id":"decent.s.05"},{"index":4,"sense_key":"decent%5:00:02:modest:01","synset_id":"decent.s.04"},{"index":5,"sense_key":"decent%5:00:00:sufficient:00","synset_id":"adequate.s.01"},{"index":6,"sense_key":"decent%3:00:00::","synset_id":"decent.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"decent%5:00:00:sufficient:00","call_ids":["senseval2.d002.s013.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"decent%5:00:00:sufficient:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":124},"cost":{"currency":"USD","total_usd":0.007280000000000001,"input_uncached_usd":0.0030800000000000003,"input_cached_usd":0.0,"output_usd":0.0042,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.779222832992673}
{"item_id":"senseval2.d002.s013.t007","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"},{"index":4,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":5,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":6,"sense_key":"education%1:09:01::","synset_id":"education.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018815,"input_uncached_usd":0.002975,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.328037415980361}
{"item_id":"senseval2.d002.s013.t013","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":2,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":3,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":4,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":5,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":6,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":9,"sense_key":"know%2:31:14::","synset_id":"know.v.11"},{"index":10,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":11,"sense_key":"know%2:31:03::","synset_id":"know.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s013.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":843,"cached_input_tokens":0,"output_tokens":181,"reasoning_output_tokens":165},"cost":{"currency":"USD","total_usd":0.009645,"input_uncached_usd":0.004215,"input_cached_usd":0.0,"output_usd":0.00543,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5662721659755334}
{"item_id":"senseval2.d002.s013.t014","gold_sense_keys":["demand%2:42:00::"],"candidates":[{"index":1,"sense_key":"demand%2:32:05::","synset_id":"demand.v.05"},{"index":2,"sense_key":"demand%2:32:00::","synset_id":"demand.v.01"},{"index":3,"sense_key":"demand%2:32:09::","synset_id":"demand.v.04"},{"index":4,"sense_key":"demand%2:42:00::","synset_id":"necessitate.v.01"},{"index":5,"sense_key":"demand%2:32:02::","synset_id":"demand.v.03"},{"index":6,"sense_key":"demand%2:32:06::","synset_id":"demand.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"demand%2:42:00::","call_ids":["senseval2.d002.s013.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"demand%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.005090000000000001,"input_uncached_usd":0.00266,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1581671671010554}
{"item_id":"senseval2.d002.s014.t000","gold_sense_keys":["commitment%1:04:00::"],"candidates":[{"index":1,"sense_key":"commitment%1:10:00::","synset_id":"commitment.n.04"},{"index":2,"sense_key":"commitment%1:04:00::","synset_id":"commitment.n.02"},{"index":3,"sense_key":"commitment%1:04:02::","synset_id":"commitment.n.03"},{"index":4,"sense_key":"commitment%1:04:01::","synset_id":"commitment.n.05"},{"index":5,"sense_key":"commitment%1:07:01::","synset_id":"committedness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"commitment%1:04:00::","call_ids":["senseval2.d002.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"commitment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":308,"reasoning_output_tokens":292},"cost":{"currency":"USD","total_usd":0.01209,"input_uncached_usd":0.00285,"input_cached_usd":0.0,"output_usd":0.00924,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.381461666082032}
{"item_id":"senseval2.d002.s014.t002","gold_sense_keys":["survive%2:42:02::"],"candidates":[{"index":1,"sense_key":"survive%2:42:01::","synset_id":"survive.v.01"},{"index":2,"sense_key":"survive%2:42:03::","synset_id":"outlive.v.01"},{"index":3,"sense_key":"survive%2:42:00::","synset_id":"exist.v.02"},{"index":4,"sense_key":"survive%2:42:02::","synset_id":"survive.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"survive%2:42:02::","call_ids":["senseval2.d002.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"survive%2:42:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":335,"reasoning_output_tokens":319},"cost":{"currency":"USD","total_usd":0.0129,"input_uncached_usd":0.00285,"input_cached_usd":0.0,"output_usd":0.01005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.335841999971308}
{"item_id":"senseval2.d002.s014.t003","gold_sense_keys":["adolescent%3:01:00::","adolescent%5:00:00:immature:01"],"candidates":[{"index":1,"sense_key":"adolescent%3:01:00::","synset_id":"adolescent.a.01"},{"index":2,"sense_key":"adolescent%5:00:00:immature:01","synset_id":"adolescent.s.03"},{"index":3,"sense_key":"adolescent%5:00:00:young:00","synset_id":"adolescent.s.01"},{"index":4,"sense_key":"adolescent%5:00:00:immature:02","synset_id":"adolescent.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"adolescent%3:01:00::","call_ids":["senseval2.d002.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"adolescent%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0185,"input_uncached_usd":0.00266,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.440977917052805}
{"item_id":"senseval2.d002.s014.t004","gold_sense_keys":["illiteracy%1:09:00::"],"candidates":[{"index":1,"sense_key":"illiteracy%1:09:00::","synset_id":"illiteracy.n.02"},{"index":2,"sense_key":"illiteracy%1:09:01::","synset_id":"illiteracy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"illiteracy%1:09:00::","call_ids":["senseval2.d002.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"illiteracy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":412,"cached_input_tokens":0,"output_tokens":339,"reasoning_output_tokens":323},"cost":{"currency":"USD","total_usd":0.012230000000000001,"input_uncached_usd":0.00206,"input_cached_usd":0.0,"output_usd":0.01017,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.281562166986987}
{"item_id":"senseval2.d002.s015.t000","gold_sense_keys":["american%3:01:01::"],"candidates":[{"index":1,"sense_key":"american%3:01:00::","synset_id":"american.a.02"},{"index":2,"sense_key":"american%3:01:01::","synset_id":"american.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"american%3:01:01::","call_ids":["senseval2.d002.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"american%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.0038450000000000003,"input_uncached_usd":0.002135,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4587099170312285}
{"item_id":"senseval2.d002.s015.t001","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":2,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":3,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":4,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":5,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"},{"index":6,"sense_key":"education%1:04:00::","synset_id":"education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":199,"reasoning_output_tokens":183},"cost":{"currency":"USD","total_usd":0.009025000000000002,"input_uncached_usd":0.0030550000000000004,"input_cached_usd":0.0,"output_usd":0.0059700000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7068250420270488}
{"item_id":"senseval2.d002.s015.t002","gold_sense_keys":["future%1:28:00::"],"candidates":[{"index":1,"sense_key":"future%1:28:00::","synset_id":"future.n.01"},{"index":2,"sense_key":"future%1:10:00::","synset_id":"future.n.02"},{"index":3,"sense_key":"future%1:06:00::","synset_id":"future.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"future%1:28:00::","call_ids":["senseval2.d002.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"future%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.003985000000000001,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1358224999858066}
{"item_id":"senseval2.d002.s015.t003","gold_sense_keys":["determine%2:31:02::"],"candidates":[{"index":1,"sense_key":"determine%2:31:02::","synset_id":"determine.v.02"},{"index":2,"sense_key":"determine%2:32:03::","synset_id":"specify.v.02"},{"index":3,"sense_key":"determine%2:31:00::","synset_id":"decide.v.01"},{"index":4,"sense_key":"determine%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"determine%2:32:05::","synset_id":"settle.v.03"},{"index":6,"sense_key":"determine%2:32:01::","synset_id":"determine.v.08"},{"index":7,"sense_key":"determine%2:31:01::","synset_id":"determine.v.03"},{"index":8,"sense_key":"determine%2:31:03::","synset_id":"determine.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"determine%2:31:02::","call_ids":["senseval2.d002.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"determine%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":79},"cost":{"currency":"USD","total_usd":0.006495000000000001,"input_uncached_usd":0.0036450000000000002,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.52768916694913}
{"item_id":"senseval2.d002.s015.t004","gold_sense_keys":["degree%1:07:00::","degree%1:26:01::"],"candidates":[{"index":1,"sense_key":"degree%1:26:01::","synset_id":"degree.n.02"},{"index":2,"sense_key":"degree%1:09:00::","synset_id":"degree.n.05"},{"index":3,"sense_key":"degree%1:10:00::","synset_id":"academic_degree.n.01"},{"index":4,"sense_key":"degree%1:23:00::","synset_id":"degree.n.04"},{"index":5,"sense_key":"degree%1:07:00::","synset_id":"degree.n.01"},{"index":6,"sense_key":"degree%1:07:01::","synset_id":"degree.n.07"},{"index":7,"sense_key":"degree%1:23:03::","synset_id":"degree.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"degree%1:07:00::","call_ids":["senseval2.d002.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"degree%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":1318,"reasoning_output_tokens":1290},"cost":{"currency":"USD","total_usd":0.04278,"input_uncached_usd":0.0032400000000000003,"input_cached_usd":0.0,"output_usd":0.03954,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":18.676294249948114}
{"item_id":"senseval2.d002.s015.t005","gold_sense_keys":["allow%2:41:00::","allow%2:42:00::"],"candidates":[{"index":1,"sense_key":"allow%2:31:01::","synset_id":"allow.v.04"},{"index":2,"sense_key":"allow%2:41:01::","synset_id":"give_up.v.11"},{"index":3,"sense_key":"allow%2:40:00::","synset_id":"allow.v.03"},{"index":4,"sense_key":"allow%2:31:00::","synset_id":"allow.v.06"},{"index":5,"sense_key":"allow%2:32:06::","synset_id":"allow.v.10"},{"index":6,"sense_key":"allow%2:41:00::","synset_id":"let.v.01"},{"index":7,"sense_key":"allow%2:42:07::","synset_id":"admit.v.05"},{"index":8,"sense_key":"allow%2:42:00::","synset_id":"leave.v.06"},{"index":9,"sense_key":"allow%2:32:00::","synset_id":"permit.v.01"},{"index":10,"sense_key":"allow%2:40:01::","synset_id":"allow.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"allow%2:41:00::","call_ids":["senseval2.d002.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"allow%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":849,"cached_input_tokens":0,"output_tokens":924,"reasoning_output_tokens":901},"cost":{"currency":"USD","total_usd":0.031965,"input_uncached_usd":0.0042450000000000005,"input_cached_usd":0.0,"output_usd":0.02772,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.672339542070404}
{"item_id":"senseval2.d002.s015.t007","gold_sense_keys":["prevail%2:33:00::"],"candidates":[{"index":1,"sense_key":"prevail%2:42:00::","synset_id":"prevail.v.03"},{"index":2,"sense_key":"prevail%2:42:02::","synset_id":"predominate.v.01"},{"index":3,"sense_key":"prevail%2:32:02::","synset_id":"prevail.v.05"},{"index":4,"sense_key":"prevail%2:33:00::","synset_id":"prevail.v.04"},{"index":5,"sense_key":"prevail%2:42:01::","synset_id":"prevail.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"prevail%2:33:00::","call_ids":["senseval2.d002.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"prevail%2:33:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":86},"cost":{"currency":"USD","total_usd":0.005825,"input_uncached_usd":0.002795,"input_cached_usd":0.0,"output_usd":0.00303,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.654554625041783}
{"item_id":"senseval2.d002.s015.t008","gold_sense_keys":["illusion%1:09:00::"],"candidates":[{"index":1,"sense_key":"illusion%1:09:00::","synset_id":"illusion.n.02"},{"index":2,"sense_key":"illusion%1:09:01::","synset_id":"illusion.n.01"},{"index":3,"sense_key":"illusion%1:04:02::","synset_id":"magic_trick.n.01"},{"index":4,"sense_key":"illusion%1:04:00::","synset_id":"delusion.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"illusion%1:09:00::","call_ids":["senseval2.d002.s015.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"illusion%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.004465,"input_uncached_usd":0.0025150000000000003,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6817952920682728}
{"item_id":"senseval2.d002.s015.t010","gold_sense_keys":["share%2:42:00::"],"candidates":[{"index":1,"sense_key":"share%2:40:00::","synset_id":"share.v.04"},{"index":2,"sense_key":"share%2:42:00::","synset_id":"share.v.01"},{"index":3,"sense_key":"share%2:32:00::","synset_id":"share.v.05"},{"index":4,"sense_key":"share%2:40:02::","synset_id":"share.v.02"},{"index":5,"sense_key":"share%2:40:01::","synset_id":"partake.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"share%2:42:00::","call_ids":["senseval2.d002.s015.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"share%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00427,"input_uncached_usd":0.00256,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5803926669759676}
{"item_id":"senseval2.d002.s016.t000","gold_sense_keys":["education%1:04:00::","education%1:04:01::"],"candidates":[{"index":1,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":2,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":3,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"},{"index":4,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:09:01::","synset_id":"education.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"education%1:04:01::","call_ids":["senseval2.d002.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"education%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":231,"reasoning_output_tokens":215},"cost":{"currency":"USD","total_usd":0.009940000000000001,"input_uncached_usd":0.00301,"input_cached_usd":0.0,"output_usd":0.00693,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.024258083081804}
{"item_id":"senseval2.d002.s016.t001","gold_sense_keys":["establishment%1:14:00::","establishment%1:14:02::"],"candidates":[{"index":1,"sense_key":"establishment%1:22:00::","synset_id":"establishment.n.06"},{"index":2,"sense_key":"establishment%1:14:01::","synset_id":"establishment.n.05"},{"index":3,"sense_key":"establishment%1:09:00::","synset_id":"establishment.n.07"},{"index":4,"sense_key":"establishment%1:04:00::","synset_id":"constitution.n.02"},{"index":5,"sense_key":"establishment%1:14:00::","synset_id":"institution.n.01"},{"index":6,"sense_key":"establishment%1:06:00::","synset_id":"establishment.n.04"},{"index":7,"sense_key":"establishment%1:14:02::","synset_id":"administration.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"establishment%1:14:01::","call_ids":["senseval2.d002.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"establishment%1:14:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":959,"reasoning_output_tokens":938},"cost":{"currency":"USD","total_usd":0.031895,"input_uncached_usd":0.003125,"input_cached_usd":0.0,"output_usd":0.02877,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":18.880638707894832}
{"item_id":"senseval2.d002.s016.t002","gold_sense_keys":["fight%2:33:01::"],"candidates":[{"index":1,"sense_key":"fight%2:41:10::","synset_id":"crusade.v.01"},{"index":2,"sense_key":"fight%2:41:00::","synset_id":"fight.v.03"},{"index":3,"sense_key":"fight%2:33:00::","synset_id":"contend.v.06"},{"index":4,"sense_key":"fight%2:33:01::","synset_id":"fight.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"fight%2:33:01::","call_ids":["senseval2.d002.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"fight%2:33:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.00595,"input_uncached_usd":0.0028000000000000004,"input_cached_usd":0.0,"output_usd":0.00315,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.28036225005053}
{"item_id":"senseval2.d002.s017.t000","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::","reason%1:16:00::"],"candidates":[{"index":1,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":2,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"},{"index":3,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":4,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":5,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":6,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reason%1:16:00::","call_ids":["senseval2.d002.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reason%1:16:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01871,"input_uncached_usd":0.00287,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.645775250042789}
{"item_id":"senseval2.d002.s017.t002","gold_sense_keys":["simple%3:00:02::","simple%5:00:00:easy:01"],"candidates":[{"index":1,"sense_key":"simple%5:00:00:naive:00","synset_id":"childlike.s.02"},{"index":2,"sense_key":"simple%5:00:00:plain:01","synset_id":"simple.s.05"},{"index":3,"sense_key":"simple%3:00:02::","synset_id":"simple.a.01"},{"index":4,"sense_key":"simple%5:00:00:retarded:00","synset_id":"dim-witted.s.01"},{"index":5,"sense_key":"simple%5:00:02:plain:01","synset_id":"bare.s.04"},{"index":6,"sense_key":"simple%3:00:01::","synset_id":"simple.a.06"},{"index":7,"sense_key":"simple%5:00:00:easy:01","synset_id":"elementary.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"simple%3:00:02::","call_ids":["senseval2.d002.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"simple%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":385,"reasoning_output_tokens":369},"cost":{"currency":"USD","total_usd":0.014535,"input_uncached_usd":0.0029850000000000002,"input_cached_usd":0.0,"output_usd":0.01155,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.841998624964617}
{"item_id":"senseval2.d002.s017.t003","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::","reason%1:16:00::"],"candidates":[{"index":1,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":2,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":3,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":4,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"},{"index":5,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":6,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"reason%1:16:00::","call_ids":["senseval2.d002.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"reason%1:16:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018705,"input_uncached_usd":0.0028650000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.800575791974552}
{"item_id":"senseval2.d002.s018.t000","gold_sense_keys":["progressive%3:00:01::"],"candidates":[{"index":1,"sense_key":"progressive%3:00:02::","synset_id":"progressive.a.03"},{"index":2,"sense_key":"progressive%5:00:00:increasing:00","synset_id":"progressive.s.02"},{"index":3,"sense_key":"progressive%3:00:01::","synset_id":"progressive.a.01"},{"index":4,"sense_key":"progressive%5:00:00:active:02","synset_id":"progressive.s.04"},{"index":5,"sense_key":"progressive%5:00:00:ordered:00","synset_id":"progressive.s.03"},{"index":6,"sense_key":"progressive%5:00:00:liberal:00","synset_id":"progressive.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"progressive%3:00:01::","call_ids":["senseval2.d002.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"progressive%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.0058000000000000005,"input_uncached_usd":0.0026200000000000004,"input_cached_usd":0.0,"output_usd":0.00318,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.963645500014536}
{"item_id":"senseval2.d002.s018.t001","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":2,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"},{"index":3,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":6,"sense_key":"education%1:09:01::","synset_id":"education.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":576,"cached_input_tokens":0,"output_tokens":224,"reasoning_output_tokens":208},"cost":{"currency":"USD","total_usd":0.009600000000000001,"input_uncached_usd":0.00288,"input_cached_usd":0.0,"output_usd":0.00672,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.463740124949254}
{"item_id":"senseval2.d002.s018.t002","gold_sense_keys":["once%4:02:01::"],"candidates":[{"index":1,"sense_key":"once%4:02:01::","synset_id":"once.r.03"},{"index":2,"sense_key":"once%4:02:02::","synset_id":"once.r.02"},{"index":3,"sense_key":"once%4:02:00::","synset_id":"once.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"once%4:02:01::","call_ids":["senseval2.d002.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"once%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":425,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.0036850000000000003,"input_uncached_usd":0.002125,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.269637249992229}
{"item_id":"senseval2.d002.s018.t003","gold_sense_keys":["call%2:32:02::"],"candidates":[{"index":1,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":2,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":3,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":4,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":5,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":6,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":7,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":8,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":9,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":10,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":11,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":12,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":13,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":14,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":15,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":16,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":17,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":18,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":19,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":20,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":21,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":22,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":23,"sense_key":"call%2:29:05::","synset_id":"call.v.28"},{"index":24,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":25,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":26,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":27,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":28,"sense_key":"call%2:32:00::","synset_id":"call.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"call%2:32:02::","call_ids":["senseval2.d002.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"call%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1486,"cached_input_tokens":0,"output_tokens":1564,"reasoning_output_tokens":1535},"cost":{"currency":"USD","total_usd":0.05435,"input_uncached_usd":0.007430000000000001,"input_cached_usd":0.0,"output_usd":0.04692,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":21.80854833300691}
{"item_id":"senseval2.d002.s018.t004","gold_sense_keys":["far%4:02:02::"],"candidates":[{"index":1,"sense_key":"far%4:02:00::","synset_id":"far.r.02"},{"index":2,"sense_key":"far%4:02:05::","synset_id":"far.r.05"},{"index":3,"sense_key":"far%4:02:02::","synset_id":"far.r.01"},{"index":4,"sense_key":"far%4:02:01::","synset_id":"far.r.04"},{"index":5,"sense_key":"far%4:02:04::","synset_id":"far.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"far%4:02:02::","call_ids":["senseval2.d002.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"far%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.004085,"input_uncached_usd":0.0025550000000000004,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3823419580003247}
{"item_id":"senseval2.d002.s018.t005","gold_sense_keys":["more%4:02:00::"],"candidates":[{"index":1,"sense_key":"more%4:02:00::","synset_id":"more.r.01"},{"index":2,"sense_key":"more%4:02:01::","synset_id":"more.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%4:02:00::","call_ids":["senseval2.d002.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":394,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.00458,"input_uncached_usd":0.00197,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6269141249358654}
{"item_id":"senseval2.d002.s018.t007","gold_sense_keys":["agreeable%3:00:00::"],"candidates":[{"index":1,"sense_key":"agreeable%3:00:00::","synset_id":"agreeable.a.01"},{"index":2,"sense_key":"agreeable%5:00:00:consistent:00","synset_id":"accordant.s.01"},{"index":3,"sense_key":"agreeable%5:00:00:accordant:00","synset_id":"agreeable.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"agreeable%3:00:00::","call_ids":["senseval2.d002.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"agreeable%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":118},"cost":{"currency":"USD","total_usd":0.006255,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.00402,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.976286499993876}
{"item_id":"senseval2.d002.s018.t008","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s018.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.00413,"input_uncached_usd":0.00185,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6670129999984056}
{"item_id":"senseval2.d002.s018.t009","gold_sense_keys":["disciplined%5:00:00:controlled:00"],"candidates":[{"index":1,"sense_key":"disciplined%5:00:00:controlled:00","synset_id":"disciplined.s.01"},{"index":2,"sense_key":"disciplined%5:00:00:trained:00","synset_id":"disciplined.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"disciplined%5:00:00:controlled:00","call_ids":["senseval2.d002.s018.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"disciplined%5:00:00:controlled:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":382,"cached_input_tokens":0,"output_tokens":240,"reasoning_output_tokens":224},"cost":{"currency":"USD","total_usd":0.00911,"input_uncached_usd":0.0019100000000000002,"input_cached_usd":0.0,"output_usd":0.0072,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.307538041030057}
{"item_id":"senseval2.d002.s018.t010","gold_sense_keys":["instruction%1:04:01::"],"candidates":[{"index":1,"sense_key":"instruction%1:10:02::","synset_id":"instruction.n.04"},{"index":2,"sense_key":"instruction%1:10:04::","synset_id":"direction.n.06"},{"index":3,"sense_key":"instruction%1:04:01::","synset_id":"education.n.01"},{"index":4,"sense_key":"instruction%1:04:00::","synset_id":"teaching.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"instruction%1:04:01::","call_ids":["senseval2.d002.s018.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"instruction%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.004235,"input_uncached_usd":0.0024950000000000003,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.426058458047919}
{"item_id":"senseval2.d002.s019.t000","gold_sense_keys":["nice%3:00:00::"],"candidates":[{"index":1,"sense_key":"nice%5:00:00:polite:00","synset_id":"courteous.s.01"},{"index":2,"sense_key":"nice%5:00:00:respectable:00","synset_id":"decent.s.01"},{"index":3,"sense_key":"nice%5:00:02:precise:00","synset_id":"nice.s.02"},{"index":4,"sense_key":"nice%5:00:00:fastidious:00","synset_id":"dainty.s.04"},{"index":5,"sense_key":"nice%3:00:00::","synset_id":"nice.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"nice%3:00:00::","call_ids":["senseval2.d002.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"nice%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":566,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.00481,"input_uncached_usd":0.00283,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4182631250005215}
{"item_id":"senseval2.d002.s019.t001","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":368,"cached_input_tokens":0,"output_tokens":41,"reasoning_output_tokens":26},"cost":{"currency":"USD","total_usd":0.00307,"input_uncached_usd":0.00184,"input_cached_usd":0.0,"output_usd":0.00123,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7603649580851197}
{"item_id":"senseval2.d002.s019.t002","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":2,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":3,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":4,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":5,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":6,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":7,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":8,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":9,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":10,"sense_key":"think%2:30:00::","synset_id":"think.v.13"},{"index":11,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":12,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":13,"sense_key":"think%2:31:12::","synset_id":"think.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"think%2:31:01::","call_ids":["senseval2.d002.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"think%2:31:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":875,"cached_input_tokens":0,"output_tokens":260,"reasoning_output_tokens":244},"cost":{"currency":"USD","total_usd":0.012175000000000002,"input_uncached_usd":0.004375,"input_cached_usd":0.0,"output_usd":0.0078000000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6535267080180347}
{"item_id":"senseval2.d002.s019.t003","gold_sense_keys":["personality%1:07:00::"],"candidates":[{"index":1,"sense_key":"personality%1:18:00::","synset_id":"personality.n.02"},{"index":2,"sense_key":"personality%1:07:00::","synset_id":"personality.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"personality%1:07:00::","call_ids":["senseval2.d002.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"personality%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":395,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.003565,"input_uncached_usd":0.001975,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4212352499598637}
{"item_id":"senseval2.d002.s019.t004","gold_sense_keys":["development%1:22:02::"],"candidates":[{"index":1,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":2,"sense_key":"development%1:10:00::","synset_id":"development.n.09"},{"index":3,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":4,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":5,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":6,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":7,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":8,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":9,"sense_key":"development%1:22:02::","synset_id":"development.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"development%1:22:02::","call_ids":["senseval2.d002.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"development%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":784,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01976,"input_uncached_usd":0.003920000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":16.108363625011407}
{"item_id":"senseval2.d002.s019.t005","gold_sense_keys":["even%4:02:01::"],"candidates":[{"index":1,"sense_key":"even%4:02:04::","synset_id":"even.r.04"},{"index":2,"sense_key":"even%4:02:01::","synset_id":"even.r.03"},{"index":3,"sense_key":"even%4:02:00::","synset_id":"even.r.01"},{"index":4,"sense_key":"even%4:02:03::","synset_id":"even.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"even%4:02:01::","call_ids":["senseval2.d002.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"even%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.005365,"input_uncached_usd":0.0023650000000000003,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0191505840048194}
{"item_id":"senseval2.d002.s019.t006","gold_sense_keys":["nice%3:00:00::"],"candidates":[{"index":1,"sense_key":"nice%3:00:00::","synset_id":"nice.a.01"},{"index":2,"sense_key":"nice%5:00:02:precise:00","synset_id":"nice.s.02"},{"index":3,"sense_key":"nice%5:00:00:fastidious:00","synset_id":"dainty.s.04"},{"index":4,"sense_key":"nice%5:00:00:respectable:00","synset_id":"decent.s.01"},{"index":5,"sense_key":"nice%5:00:00:polite:00","synset_id":"courteous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nice%3:00:00::","call_ids":["senseval2.d002.s019.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nice%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.005265000000000001,"input_uncached_usd":0.0028350000000000003,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.048935957951471}
{"item_id":"senseval2.d002.s019.t007","gold_sense_keys":["minimize%2:30:00::"],"candidates":[{"index":1,"sense_key":"minimize%2:32:00::","synset_id":"minimize.v.03"},{"index":2,"sense_key":"minimize%2:30:00::","synset_id":"minimize.v.01"},{"index":3,"sense_key":"minimize%2:32:01::","synset_id":"understate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"minimize%2:30:00::","call_ids":["senseval2.d002.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"minimize%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":1045,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.033435000000000006,"input_uncached_usd":0.002085,"input_cached_usd":0.0,"output_usd":0.03135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":17.676415666937828}
{"item_id":"senseval2.d002.s019.t009","gold_sense_keys":["test%1:10:00::"],"candidates":[{"index":1,"sense_key":"test%1:05:00::","synset_id":"test.n.06"},{"index":2,"sense_key":"test%1:04:00::","synset_id":"test.n.05"},{"index":3,"sense_key":"test%1:09:02::","synset_id":"trial.n.02"},{"index":4,"sense_key":"test%1:04:02::","synset_id":"test.n.02"},{"index":5,"sense_key":"test%1:10:00::","synset_id":"examination.n.02"},{"index":6,"sense_key":"test%1:04:03::","synset_id":"test.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"test%1:10:00::","call_ids":["senseval2.d002.s019.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"test%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":314,"reasoning_output_tokens":298},"cost":{"currency":"USD","total_usd":0.012375,"input_uncached_usd":0.002955,"input_cached_usd":0.0,"output_usd":0.00942,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.439302291953936}
{"item_id":"senseval2.d002.s019.t010","gold_sense_keys":["often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:02::","synset_id":"often.r.03"},{"index":2,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":3,"sense_key":"often%4:02:01::","synset_id":"much.r.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"often%4:02:02::","call_ids":["senseval2.d002.s019.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"often%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":88},"cost":{"currency":"USD","total_usd":0.005180000000000001,"input_uncached_usd":0.0020900000000000003,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5926269999472424}
{"item_id":"senseval2.d002.s019.t012","gold_sense_keys":["result%1:11:00::"],"candidates":[{"index":1,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":2,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"},{"index":3,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":4,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"result%1:11:00::","call_ids":["senseval2.d002.s019.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"result%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":166,"reasoning_output_tokens":150},"cost":{"currency":"USD","total_usd":0.007560000000000001,"input_uncached_usd":0.0025800000000000003,"input_cached_usd":0.0,"output_usd":0.00498,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7170954999746755}
{"item_id":"senseval2.d002.s020.t001","gold_sense_keys":["provide%2:40:00::"],"candidates":[{"index":1,"sense_key":"provide%2:42:00::","synset_id":"leave.v.06"},{"index":2,"sense_key":"provide%2:30:05::","synset_id":"provide.v.07"},{"index":3,"sense_key":"provide%2:40:00::","synset_id":"supply.v.01"},{"index":4,"sense_key":"provide%2:32:00::","synset_id":"provide.v.03"},{"index":5,"sense_key":"provide%2:41:00::","synset_id":"put_up.v.02"},{"index":6,"sense_key":"provide%2:34:00::","synset_id":"provide.v.02"},{"index":7,"sense_key":"provide%2:40:08::","synset_id":"provide.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"provide%2:40:00::","call_ids":["senseval2.d002.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"provide%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":101},"cost":{"currency":"USD","total_usd":0.0068200000000000005,"input_uncached_usd":0.0033100000000000004,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.145863957935944}
{"item_id":"senseval2.d002.s020.t002","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":28},"cost":{"currency":"USD","total_usd":0.00317,"input_uncached_usd":0.00185,"input_cached_usd":0.0,"output_usd":0.00132,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6415325000416487}
{"item_id":"senseval2.d002.s020.t003","gold_sense_keys":["profession%1:04:00::","profession%1:14:00::"],"candidates":[{"index":1,"sense_key":"profession%1:14:00::","synset_id":"profession.n.01"},{"index":2,"sense_key":"profession%1:10:01::","synset_id":"profession.n.04"},{"index":3,"sense_key":"profession%1:10:00::","synset_id":"profession.n.03"},{"index":4,"sense_key":"profession%1:04:00::","synset_id":"profession.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"profession%1:04:00::","call_ids":["senseval2.d002.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"profession%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":399,"reasoning_output_tokens":383},"cost":{"currency":"USD","total_usd":0.014245,"input_uncached_usd":0.002275,"input_cached_usd":0.0,"output_usd":0.01197,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.27100812504068}
{"item_id":"senseval2.d002.s020.t004","gold_sense_keys":["pass%2:41:08::"],"candidates":[{"index":1,"sense_key":"pass%2:40:00::","synset_id":"pass.v.05"},{"index":2,"sense_key":"pass%2:42:00::","synset_id":"spend.v.01"},{"index":3,"sense_key":"pass%2:38:03::","synset_id":"elapse.v.01"},{"index":4,"sense_key":"pass%2:42:08::","synset_id":"run.v.03"},{"index":5,"sense_key":"pass%2:41:04::","synset_id":"pass.v.09"},{"index":6,"sense_key":"pass%2:38:05::","synset_id":"travel_by.v.01"},{"index":7,"sense_key":"pass%2:41:02::","synset_id":"legislate.v.01"},{"index":8,"sense_key":"pass%2:35:00::","synset_id":"guide.v.05"},{"index":9,"sense_key":"pass%2:41:11::","synset_id":"pass.v.17"},{"index":10,"sense_key":"pass%2:30:00::","synset_id":"happen.v.01"},{"index":11,"sense_key":"pass%2:32:00::","synset_id":"authorize.v.01"},{"index":12,"sense_key":"pass%2:42:04::","synset_id":"exceed.v.02"},{"index":13,"sense_key":"pass%2:32:01::","synset_id":"communicate.v.01"},{"index":14,"sense_key":"pass%2:38:02::","synset_id":"sink.v.03"},{"index":15,"sense_key":"pass%2:38:04::","synset_id":"pass.v.22"},{"index":16,"sense_key":"pass%2:40:01::","synset_id":"pass.v.20"},{"index":17,"sense_key":"pass%2:30:01::","synset_id":"evanesce.v.01"},{"index":18,"sense_key":"pass%2:38:01::","synset_id":"pass.v.07"},{"index":19,"sense_key":"pass%2:29:00::","synset_id":"excrete.v.01"},{"index":20,"sense_key":"pass%2:38:00::","synset_id":"pass.v.01"},{"index":21,"sense_key":"pass%2:30:11::","synset_id":"die.v.01"},{"index":22,"sense_key":"pass%2:41:07::","synset_id":"pass.v.16"},{"index":23,"sense_key":"pass%2:40:02::","synset_id":"pass.v.18"},{"index":24,"sense_key":"pass%2:40:13::","synset_id":"fall.v.21"},{"index":25,"sense_key":"pass%2:41:08::","synset_id":"pass.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":25,"chosen_sense_key":"pass%2:41:08::","call_ids":["senseval2.d002.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":25,"predicted_sense_key":"pass%2:41:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1448,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.010690000000000002,"input_uncached_usd":0.007240000000000001,"input_cached_usd":0.0,"output_usd":0.00345,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.706340417033061}
{"item_id":"senseval2.d002.s020.t005","gold_sense_keys":["course%1:04:01::"],"candidates":[{"index":1,"sense_key":"course%1:14:00::","synset_id":"course.n.02"},{"index":2,"sense_key":"course%1:14:01::","synset_id":"class.n.02"},{"index":3,"sense_key":"course%1:04:00::","synset_id":"course.n.04"},{"index":4,"sense_key":"course%1:13:00::","synset_id":"course.n.07"},{"index":5,"sense_key":"course%1:06:01::","synset_id":"course.n.08"},{"index":6,"sense_key":"course%1:04:01::","synset_id":"course.n.01"},{"index":7,"sense_key":"course%1:17:00::","synset_id":"path.n.04"},{"index":8,"sense_key":"course%1:15:00::","synset_id":"course.n.03"},{"index":9,"sense_key":"course%1:06:00::","synset_id":"course.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"course%1:04:01::","call_ids":["senseval2.d002.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"course%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.00578,"input_uncached_usd":0.0036200000000000004,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0013844160130247}
{"item_id":"senseval2.d002.s020.t006","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"},{"index":2,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s020.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":373,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.003725,"input_uncached_usd":0.0018650000000000001,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7597229580860585}
{"item_id":"senseval2.d002.s020.t008","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s020.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":373,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.004115,"input_uncached_usd":0.0018650000000000001,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1370308329351246}
{"item_id":"senseval2.d002.s020.t009","gold_sense_keys":["philosophy%1:09:00::"],"candidates":[{"index":1,"sense_key":"philosophy%1:09:01::","synset_id":"doctrine.n.01"},{"index":2,"sense_key":"philosophy%1:09:00::","synset_id":"philosophy.n.02"},{"index":3,"sense_key":"philosophy%1:09:02::","synset_id":"philosophy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"philosophy%1:09:00::","call_ids":["senseval2.d002.s020.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"philosophy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":187,"reasoning_output_tokens":171},"cost":{"currency":"USD","total_usd":0.007835,"input_uncached_usd":0.002225,"input_cached_usd":0.0,"output_usd":0.00561,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.122445666929707}
{"item_id":"senseval2.d002.s021.t000","gold_sense_keys":["take%2:31:02::"],"candidates":[{"index":1,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":3,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":4,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":5,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":6,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":7,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":8,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":9,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":12,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":13,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":14,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"},{"index":15,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":16,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":17,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":18,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":21,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":22,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":23,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":24,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":25,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":26,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":27,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":28,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":29,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":30,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":31,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":34,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":35,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":36,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":37,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":38,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":39,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":40,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":41,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":42,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"take%2:31:02::","call_ids":["senseval2.d002.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"take%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2153,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.012685,"input_uncached_usd":0.010765,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9723992500221357}
{"item_id":"senseval2.d002.s021.t002","gold_sense_keys":["course%1:04:01::"],"candidates":[{"index":1,"sense_key":"course%1:14:01::","synset_id":"class.n.02"},{"index":2,"sense_key":"course%1:15:00::","synset_id":"course.n.03"},{"index":3,"sense_key":"course%1:17:00::","synset_id":"path.n.04"},{"index":4,"sense_key":"course%1:14:00::","synset_id":"course.n.02"},{"index":5,"sense_key":"course%1:13:00::","synset_id":"course.n.07"},{"index":6,"sense_key":"course%1:04:01::","synset_id":"course.n.01"},{"index":7,"sense_key":"course%1:04:00::","synset_id":"course.n.04"},{"index":8,"sense_key":"course%1:06:01::","synset_id":"course.n.08"},{"index":9,"sense_key":"course%1:06:00::","synset_id":"course.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"course%1:04:01::","call_ids":["senseval2.d002.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"course%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":704,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.00541,"input_uncached_usd":0.00352,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.185515459044836}
{"item_id":"senseval2.d002.s021.t003","gold_sense_keys":["college%1:14:01::"],"candidates":[{"index":1,"sense_key":"college%1:14:00::","synset_id":"college.n.01"},{"index":2,"sense_key":"college%1:14:01::","synset_id":"college.n.02"},{"index":3,"sense_key":"college%1:06:00::","synset_id":"college.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"college%1:14:01::","call_ids":["senseval2.d002.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"college%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":378,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.0041400000000000005,"input_uncached_usd":0.0018900000000000002,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7924838330363855}
{"item_id":"senseval2.d002.s021.t004","gold_sense_keys":["think%2:31:03::","think%2:36:00::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":3,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":4,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":5,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":6,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":7,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":8,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":11,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":12,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval2.d002.s021.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":857,"cached_input_tokens":0,"output_tokens":210,"reasoning_output_tokens":194},"cost":{"currency":"USD","total_usd":0.010585,"input_uncached_usd":0.004285000000000001,"input_cached_usd":0.0,"output_usd":0.0063,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6589229999808595}
{"item_id":"senseval2.d002.s022.t001","gold_sense_keys":["fairly%4:02:00::"],"candidates":[{"index":1,"sense_key":"fairly%4:02:02::","synset_id":"reasonably.r.01"},{"index":2,"sense_key":"fairly%4:02:03::","synset_id":"fairly.r.03"},{"index":3,"sense_key":"fairly%4:02:00::","synset_id":"fairly.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fairly%4:02:02::","call_ids":["senseval2.d002.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fairly%4:02:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":430,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01799,"input_uncached_usd":0.00215,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.242786500020884}
{"item_id":"senseval2.d002.s022.t002","gold_sense_keys":["describe%2:32:00::"],"candidates":[{"index":1,"sense_key":"describe%2:31:00::","synset_id":"identify.v.05"},{"index":2,"sense_key":"describe%2:32:01::","synset_id":"report.v.01"},{"index":3,"sense_key":"describe%2:32:00::","synset_id":"describe.v.01"},{"index":4,"sense_key":"describe%2:35:00::","synset_id":"trace.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"describe%2:32:01::","call_ids":["senseval2.d002.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"describe%2:32:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018195,"input_uncached_usd":0.0023550000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.17870116699487}
{"item_id":"senseval2.d002.s022.t003","gold_sense_keys":["pap%1:10:00::"],"candidates":[{"index":1,"sense_key":"pap%1:08:00::","synset_id":"nipple.n.01"},{"index":2,"sense_key":"pap%1:10:00::","synset_id":"pap.n.01"},{"index":3,"sense_key":"pap%1:13:00::","synset_id":"soft_diet.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pap%1:10:00::","call_ids":["senseval2.d002.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pap%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.00411,"input_uncached_usd":0.00198,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6887903329916298}
{"item_id":"senseval2.d002.s022.t004","gold_sense_keys":["course%1:04:01::"],"candidates":[{"index":1,"sense_key":"course%1:06:01::","synset_id":"course.n.08"},{"index":2,"sense_key":"course%1:14:00::","synset_id":"course.n.02"},{"index":3,"sense_key":"course%1:13:00::","synset_id":"course.n.07"},{"index":4,"sense_key":"course%1:14:01::","synset_id":"class.n.02"},{"index":5,"sense_key":"course%1:04:01::","synset_id":"course.n.01"},{"index":6,"sense_key":"course%1:04:00::","synset_id":"course.n.04"},{"index":7,"sense_key":"course%1:15:00::","synset_id":"course.n.03"},{"index":8,"sense_key":"course%1:17:00::","synset_id":"path.n.04"},{"index":9,"sense_key":"course%1:06:00::","synset_id":"course.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"course%1:04:01::","call_ids":["senseval2.d002.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"course%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":704,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.005770000000000001,"input_uncached_usd":0.00352,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.17576337500941}
{"item_id":"senseval2.d002.s023.t002","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":355,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.0031550000000000003,"input_uncached_usd":0.001775,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.420157666085288}
{"item_id":"senseval2.d002.s023.t003","gold_sense_keys":["distinct%5:00:00:different:00"],"candidates":[{"index":1,"sense_key":"distinct%5:00:00:separate:00","synset_id":"discrete.s.01"},{"index":2,"sense_key":"distinct%5:00:00:definite:00","synset_id":"distinct.s.03"},{"index":3,"sense_key":"distinct%3:00:00::","synset_id":"distinct.a.02"},{"index":4,"sense_key":"distinct%5:00:00:different:00","synset_id":"distinct.s.01"},{"index":5,"sense_key":"distinct%5:00:00:clear:00","synset_id":"clear-cut.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"distinct%5:00:00:different:00","call_ids":["senseval2.d002.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"distinct%5:00:00:different:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":140},"cost":{"currency":"USD","total_usd":0.00742,"input_uncached_usd":0.0027400000000000002,"input_cached_usd":0.0,"output_usd":0.00468,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9540320839732885}
{"item_id":"senseval2.d002.s023.t004","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":359,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.004405,"input_uncached_usd":0.0017950000000000002,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7031307920115069}
{"item_id":"senseval2.d002.s023.t005","gold_sense_keys":["establishment%1:14:00::","establishment%1:14:02::"],"candidates":[{"index":1,"sense_key":"establishment%1:14:01::","synset_id":"establishment.n.05"},{"index":2,"sense_key":"establishment%1:14:02::","synset_id":"administration.n.02"},{"index":3,"sense_key":"establishment%1:14:00::","synset_id":"institution.n.01"},{"index":4,"sense_key":"establishment%1:06:00::","synset_id":"establishment.n.04"},{"index":5,"sense_key":"establishment%1:22:00::","synset_id":"establishment.n.06"},{"index":6,"sense_key":"establishment%1:09:00::","synset_id":"establishment.n.07"},{"index":7,"sense_key":"establishment%1:04:00::","synset_id":"constitution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"establishment%1:14:02::","call_ids":["senseval2.d002.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"establishment%1:14:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":974,"reasoning_output_tokens":952},"cost":{"currency":"USD","total_usd":0.03214,"input_uncached_usd":0.0029200000000000003,"input_cached_usd":0.0,"output_usd":0.02922,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":16.69804541603662}
{"item_id":"senseval2.d002.s024.t000","gold_sense_keys":["know%2:31:00::"],"candidates":[{"index":1,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":2,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":3,"sense_key":"know%2:31:14::","synset_id":"know.v.11"},{"index":4,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":5,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":6,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":7,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":8,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":9,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":10,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":11,"sense_key":"know%2:31:04::","synset_id":"know.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"know%2:31:00::","call_ids":["senseval2.d002.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"know%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.006035,"input_uncached_usd":0.0040550000000000004,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.366151374997571}
{"item_id":"senseval2.d002.s024.t004","gold_sense_keys":["seriously%4:02:00::"],"candidates":[{"index":1,"sense_key":"seriously%4:02:01::","synset_id":"badly.r.01"},{"index":2,"sense_key":"seriously%4:02:00::","synset_id":"seriously.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"seriously%4:02:00::","call_ids":["senseval2.d002.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"seriously%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":392,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":96},"cost":{"currency":"USD","total_usd":0.00529,"input_uncached_usd":0.0019600000000000004,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.359419292071834}
{"item_id":"senseval2.d002.s024.t005","gold_sense_keys":["committed%3:00:00::"],"candidates":[{"index":1,"sense_key":"committed%3:00:04::","synset_id":"attached.a.03"},{"index":2,"sense_key":"committed%3:00:00::","synset_id":"committed.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"committed%3:00:00::","call_ids":["senseval2.d002.s024.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"committed%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":372,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00354,"input_uncached_usd":0.00186,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.436951416078955}
{"item_id":"senseval2.d002.s024.t007","gold_sense_keys":["teaching%1:04:00::","teaching%1:04:01::"],"candidates":[{"index":1,"sense_key":"teaching%1:04:00::","synset_id":"teaching.n.01"},{"index":2,"sense_key":"teaching%1:09:00::","synset_id":"teaching.n.02"},{"index":3,"sense_key":"teaching%1:04:01::","synset_id":"education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"teaching%1:04:01::","call_ids":["senseval2.d002.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"teaching%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.00451,"input_uncached_usd":0.0022600000000000003,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1336236658971757}
{"item_id":"senseval2.d002.s025.t000","gold_sense_keys":["best%3:00:00::"],"candidates":[{"index":1,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":2,"sense_key":"good%5:00:00:healthy:00","synset_id":"good.s.15"},{"index":3,"sense_key":"good%5:00:00:discriminating:00","synset_id":"good.s.19"},{"index":4,"sense_key":"estimable%5:00:00:reputable:00","synset_id":"estimable.s.01"},{"index":5,"sense_key":"good%5:00:00:healthful:00","synset_id":"good.s.16"},{"index":6,"sense_key":"best%5:00:00:advisable:00","synset_id":"better.s.01"},{"index":7,"sense_key":"good%5:00:00:nice:00","synset_id":"good.s.04"},{"index":8,"sense_key":"good%3:00:02::","synset_id":"good.a.03"},{"index":9,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":10,"sense_key":"good%5:00:00:opportune:00","synset_id":"good.s.10"},{"index":11,"sense_key":"beneficial%5:00:00:advantageous:00","synset_id":"beneficial.s.01"},{"index":12,"sense_key":"good%5:00:00:pleasing:00","synset_id":"good.s.13"},{"index":13,"sense_key":"adept%5:00:00:skilled:00","synset_id":"adept.s.01"},{"index":14,"sense_key":"good%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":15,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":16,"sense_key":"good%5:00:00:fresh:01","synset_id":"good.s.18"},{"index":17,"sense_key":"good%5:00:00:righteous:00","synset_id":"good.s.05"},{"index":18,"sense_key":"best%3:00:00::","synset_id":"best.a.01"},{"index":19,"sense_key":"good%3:00:01::","synset_id":"good.a.01"},{"index":20,"sense_key":"good%5:00:00:complete:00","synset_id":"good.s.07"},{"index":21,"sense_key":"dependable%5:00:00:sound:00","synset_id":"dependable.s.03"},{"index":22,"sense_key":"good%5:00:00:genuine:00","synset_id":"good.s.17"},{"index":23,"sense_key":"dear%5:00:00:close:02","synset_id":"dear.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":18,"chosen_sense_key":"best%3:00:00::","call_ids":["senseval2.d002.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":18,"predicted_sense_key":"best%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1314,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.01005,"input_uncached_usd":0.00657,"input_cached_usd":0.0,"output_usd":0.00348,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3381221250165254}
{"item_id":"senseval2.d002.s025.t001","gold_sense_keys":["bright%5:00:00:intelligent:00"],"candidates":[{"index":1,"sense_key":"bright%5:00:00:glorious:00","synset_id":"bright.s.05"},{"index":2,"sense_key":"bright%5:00:00:reverberant:00","synset_id":"bright.s.06"},{"index":3,"sense_key":"bright%5:00:00:intelligent:00","synset_id":"bright.s.02"},{"index":4,"sense_key":"bright%3:00:00::","synset_id":"bright.a.01"},{"index":5,"sense_key":"bright%5:00:00:auspicious:00","synset_id":"bright.s.08"},{"index":6,"sense_key":"bright%5:00:00:light:06","synset_id":"bright.s.03"},{"index":7,"sense_key":"bright%5:00:00:polished:00","synset_id":"bright.s.04"},{"index":8,"sense_key":"bright%5:00:00:happy:00","synset_id":"bright.s.07"},{"index":9,"sense_key":"bright%5:00:00:colorful:00","synset_id":"bright.s.01"},{"index":10,"sense_key":"bright%3:00:02::","synset_id":"undimmed.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bright%5:00:00:intelligent:00","call_ids":["senseval2.d002.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bright%5:00:00:intelligent:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":756,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.005730000000000001,"input_uncached_usd":0.0037800000000000004,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0057130840141326}
{"item_id":"senseval2.d002.s025.t002","gold_sense_keys":["generation%1:14:01::"],"candidates":[{"index":1,"sense_key":"generation%1:04:01::","synset_id":"generation.n.06"},{"index":2,"sense_key":"generation%1:28:00::","synset_id":"generation.n.03"},{"index":3,"sense_key":"generation%1:11:00::","synset_id":"genesis.n.01"},{"index":4,"sense_key":"generation%1:14:00::","synset_id":"generation.n.02"},{"index":5,"sense_key":"generation%1:04:00::","synset_id":"generation.n.07"},{"index":6,"sense_key":"generation%1:28:01::","synset_id":"generation.n.04"},{"index":7,"sense_key":"generation%1:14:01::","synset_id":"coevals.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"generation%1:14:01::","call_ids":["senseval2.d002.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"generation%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.004915000000000001,"input_uncached_usd":0.0025450000000000004,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2041657919762656}
{"item_id":"senseval2.d002.s025.t003","gold_sense_keys":["very%4:02:00::"],"candidates":[{"index":1,"sense_key":"very%4:02:00::","synset_id":"very.r.01"},{"index":2,"sense_key":"very%4:02:01::","synset_id":"very.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"very%4:02:00::","call_ids":["senseval2.d002.s025.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"very%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":373,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.004655,"input_uncached_usd":0.0018650000000000001,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1716757089598104}
{"item_id":"senseval2.d002.s025.t006","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":3,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"},{"index":4,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval2.d002.s025.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.004425,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.984915916924365}
{"item_id":"senseval2.d002.s026.t000","gold_sense_keys":["job%1:04:00::"],"candidates":[{"index":1,"sense_key":"job%1:04:01::","synset_id":"job.n.02"},{"index":2,"sense_key":"job%1:10:01::","synset_id":"job.n.11"},{"index":3,"sense_key":"job%1:04:04::","synset_id":"job.n.05"},{"index":4,"sense_key":"job%1:18:00::","synset_id":"job.n.09"},{"index":5,"sense_key":"job%1:04:02::","synset_id":"caper.n.03"},{"index":6,"sense_key":"job%1:26:00::","synset_id":"problem.n.01"},{"index":7,"sense_key":"job%1:04:05::","synset_id":"job.n.06"},{"index":8,"sense_key":"job%1:18:01::","synset_id":"job.n.10"},{"index":9,"sense_key":"job%1:06:00::","synset_id":"job.n.03"},{"index":10,"sense_key":"job%1:04:03::","synset_id":"job.n.07"},{"index":11,"sense_key":"job%1:06:01::","synset_id":"job.n.04"},{"index":12,"sense_key":"job%1:04:00::","synset_id":"occupation.n.01"},{"index":13,"sense_key":"job%1:10:00::","synset_id":"job.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"job%1:04:00::","call_ids":["senseval2.d002.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"job%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":814,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01991,"input_uncached_usd":0.004070000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.580844582989812}
{"item_id":"senseval2.d002.s026.t001","gold_sense_keys":["well%4:02:00::"],"candidates":[{"index":1,"sense_key":"well%4:02:13::","synset_id":"well.r.09"},{"index":2,"sense_key":"well%4:02:10::","synset_id":"well.r.11"},{"index":3,"sense_key":"well%4:02:03::","synset_id":"well.r.03"},{"index":4,"sense_key":"well%4:02:08::","synset_id":"well.r.05"},{"index":5,"sense_key":"well%4:02:09::","synset_id":"well.r.12"},{"index":6,"sense_key":"well%4:02:01::","synset_id":"well.r.13"},{"index":7,"sense_key":"well%4:02:15::","synset_id":"well.r.04"},{"index":8,"sense_key":"well%4:02:12::","synset_id":"well.r.10"},{"index":9,"sense_key":"well%4:02:05::","synset_id":"well.r.02"},{"index":10,"sense_key":"well%4:02:14::","synset_id":"well.r.08"},{"index":11,"sense_key":"well%4:02:07::","synset_id":"well.r.06"},{"index":12,"sense_key":"well%4:02:00::","synset_id":"well.r.01"},{"index":13,"sense_key":"well%4:02:11::","synset_id":"well.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"well%4:02:00::","call_ids":["senseval2.d002.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"well%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":838,"cached_input_tokens":0,"output_tokens":944,"reasoning_output_tokens":923},"cost":{"currency":"USD","total_usd":0.03251,"input_uncached_usd":0.00419,"input_cached_usd":0.0,"output_usd":0.02832,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":16.435722999973223}
{"item_id":"senseval2.d002.s027.t000","gold_sense_keys":["remember%2:31:00::"],"candidates":[{"index":1,"sense_key":"remember%2:31:08::","synset_id":"remember.v.07"},{"index":2,"sense_key":"remember%2:31:02::","synset_id":"remember.v.03"},{"index":3,"sense_key":"remember%2:31:01::","synset_id":"remember.v.02"},{"index":4,"sense_key":"remember%2:31:00::","synset_id":"remember.v.01"},{"index":5,"sense_key":"remember%2:32:07::","synset_id":"remember.v.05"},{"index":6,"sense_key":"remember%2:32:00::","synset_id":"commend.v.05"},{"index":7,"sense_key":"remember%2:40:00::","synset_id":"remember.v.04"},{"index":8,"sense_key":"remember%2:31:09::","synset_id":"commemorate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"remember%2:31:00::","call_ids":["senseval2.d002.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"remember%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.006365,"input_uncached_usd":0.003125,"input_cached_usd":0.0,"output_usd":0.0032400000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.293862667051144}
{"item_id":"senseval2.d002.s027.t001","gold_sense_keys":["truly%4:02:00::","truly%4:02:04::"],"candidates":[{"index":1,"sense_key":"truly%4:02:03::","synset_id":"sincerely.r.01"},{"index":2,"sense_key":"truly%4:02:01::","synset_id":"rightfully.r.01"},{"index":3,"sense_key":"truly%4:02:00::","synset_id":"truly.r.01"},{"index":4,"sense_key":"truly%4:02:04::","synset_id":"in_truth.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"truly%4:02:00::","call_ids":["senseval2.d002.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"truly%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":249,"reasoning_output_tokens":233},"cost":{"currency":"USD","total_usd":0.009775,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.00747,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.235761583899148}
{"item_id":"senseval2.d002.s027.t003","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"},{"index":2,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s027.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":327,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.003075,"input_uncached_usd":0.0016350000000000002,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5044795830035582}
{"item_id":"senseval2.d002.s028.t000","gold_sense_keys":["education%1:09:01::"],"candidates":[{"index":1,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"},{"index":2,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":3,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":6,"sense_key":"education%1:07:01::","synset_id":"education.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s028.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"education%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":347,"reasoning_output_tokens":331},"cost":{"currency":"USD","total_usd":0.013085,"input_uncached_usd":0.0026750000000000003,"input_cached_usd":0.0,"output_usd":0.01041,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.4318368340609595}
{"item_id":"senseval2.d002.s028.t001","gold_sense_keys":["proceed%2:38:00::"],"candidates":[{"index":1,"sense_key":"proceed%2:32:00::","synset_id":"continue.v.02"},{"index":2,"sense_key":"proceed%2:42:00::","synset_id":"continue.v.01"},{"index":3,"sense_key":"proceed%2:38:00::","synset_id":"proceed.v.02"},{"index":4,"sense_key":"proceed%2:41:00::","synset_id":"go.v.02"},{"index":5,"sense_key":"proceed%2:30:00::","synset_id":"proceed.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"proceed%2:38:00::","call_ids":["senseval2.d002.s028.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"proceed%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":408,"reasoning_output_tokens":392},"cost":{"currency":"USD","total_usd":0.01479,"input_uncached_usd":0.00255,"input_cached_usd":0.0,"output_usd":0.012240000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.204667416983284}
{"item_id":"senseval2.d002.s028.t003","gold_sense_keys":["competent%3:00:00::"],"candidates":[{"index":1,"sense_key":"competent%5:00:00:adequate:00","synset_id":"competent.s.01"},{"index":2,"sense_key":"competent%3:00:00::","synset_id":"competent.a.01"},{"index":3,"sense_key":"competent%3:00:02::","synset_id":"competent.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"competent%3:00:00::","call_ids":["senseval2.d002.s028.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"competent%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":365,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017665,"input_uncached_usd":0.0018250000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.208754042047076}
{"item_id":"senseval2.d002.s029.t000","gold_sense_keys":["sense%1:10:00::"],"candidates":[{"index":1,"sense_key":"sense%1:09:02::","synset_id":"sense.n.03"},{"index":2,"sense_key":"sense%1:09:06::","synset_id":"sense.n.05"},{"index":3,"sense_key":"sense%1:09:04::","synset_id":"common_sense.n.01"},{"index":4,"sense_key":"sense%1:09:05::","synset_id":"sense.n.01"},{"index":5,"sense_key":"sense%1:10:00::","synset_id":"sense.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"sense%1:10:00::","call_ids":["senseval2.d002.s029.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"sense%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.004515,"input_uncached_usd":0.002565,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9552678329637274}
{"item_id":"senseval2.d002.s029.t001","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"},{"index":2,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s029.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":331,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.003095,"input_uncached_usd":0.0016550000000000002,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.493614083970897}
{"item_id":"senseval2.d002.s029.t002","gold_sense_keys":["compare%2:31:01::"],"candidates":[{"index":1,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":2,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"},{"index":3,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":4,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"compare%2:31:01::","call_ids":["senseval2.d002.s029.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"compare%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00412,"input_uncached_usd":0.0022,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1733237500302494}
{"item_id":"senseval2.d002.s029.t003","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval2.d002.s029.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":159,"reasoning_output_tokens":144},"cost":{"currency":"USD","total_usd":0.00662,"input_uncached_usd":0.00185,"input_cached_usd":0.0,"output_usd":0.00477,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4646757079754025}
{"item_id":"senseval2.d002.s030.t000","gold_sense_keys":["brilliant%5:00:00:intelligent:00"],"candidates":[{"index":1,"sense_key":"brilliant%5:00:00:intelligent:00","synset_id":"brainy.s.01"},{"index":2,"sense_key":"brilliant%5:00:00:reverberant:00","synset_id":"bright.s.06"},{"index":3,"sense_key":"brilliant%5:00:00:superior:02","synset_id":"brilliant.s.01"},{"index":4,"sense_key":"brilliant%5:00:00:impressive:00","synset_id":"brilliant.s.03"},{"index":5,"sense_key":"brilliant%5:00:00:bright:00","synset_id":"brilliant.s.05"},{"index":6,"sense_key":"brilliant%5:00:00:colorful:00","synset_id":"bright.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brilliant%5:00:00:intelligent:00","call_ids":["senseval2.d002.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brilliant%5:00:00:intelligent:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":275,"reasoning_output_tokens":259},"cost":{"currency":"USD","total_usd":0.01091,"input_uncached_usd":0.00266,"input_cached_usd":0.0,"output_usd":0.00825,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.224611875019036}
{"item_id":"senseval2.d002.s030.t001","gold_sense_keys":["probably%4:02:00::"],"candidates":[{"index":1,"sense_key":"probably%4:02:00::","synset_id":"probably.r.01"},{"index":2,"sense_key":"probably%4:02:01::","synset_id":"credibly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"probably%4:02:00::","call_ids":["senseval2.d002.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"probably%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":375,"cached_input_tokens":0,"output_tokens":169,"reasoning_output_tokens":154},"cost":{"currency":"USD","total_usd":0.006945,"input_uncached_usd":0.0018750000000000001,"input_cached_usd":0.0,"output_usd":0.00507,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.441831915988587}
{"item_id":"senseval2.d002.s031.t000","gold_sense_keys":["competent%3:00:00::"],"candidates":[{"index":1,"sense_key":"competent%5:00:00:adequate:00","synset_id":"competent.s.01"},{"index":2,"sense_key":"competent%3:00:02::","synset_id":"competent.a.03"},{"index":3,"sense_key":"competent%3:00:00::","synset_id":"competent.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"competent%3:00:00::","call_ids":["senseval2.d002.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"competent%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":347,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017575,"input_uncached_usd":0.0017350000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.48673420806881}
{"item_id":"senseval2.d002.s031.t002","gold_sense_keys":["serve%2:42:02::","serve%2:42:03::"],"candidates":[{"index":1,"sense_key":"serve%2:34:00::","synset_id":"serve.v.06"},{"index":2,"sense_key":"serve%2:42:12::","synset_id":"suffice.v.01"},{"index":3,"sense_key":"serve%2:41:13::","synset_id":"serve.v.13"},{"index":4,"sense_key":"serve%2:41:00::","synset_id":"service.v.01"},{"index":5,"sense_key":"serve%2:34:01::","synset_id":"serve.v.05"},{"index":6,"sense_key":"serve%2:35:00::","synset_id":"serve.v.11"},{"index":7,"sense_key":"serve%2:41:01::","synset_id":"serve.v.10"},{"index":8,"sense_key":"serve%2:42:03::","synset_id":"serve.v.01"},{"index":9,"sense_key":"serve%2:33:00::","synset_id":"serve.v.02"},{"index":10,"sense_key":"serve%2:42:01::","synset_id":"serve.v.03"},{"index":11,"sense_key":"serve%2:35:01::","synset_id":"serve.v.14"},{"index":12,"sense_key":"serve%2:42:02::","synset_id":"serve.v.08"},{"index":13,"sense_key":"serve%2:41:02::","synset_id":"serve.v.07"},{"index":14,"sense_key":"serve%2:42:00::","synset_id":"serve.v.09"},{"index":15,"sense_key":"serve%2:33:01::","synset_id":"serve.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"serve%2:42:02::","call_ids":["senseval2.d002.s031.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"serve%2:42:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":920,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":123},"cost":{"currency":"USD","total_usd":0.00877,"input_uncached_usd":0.004600000000000001,"input_cached_usd":0.0,"output_usd":0.00417,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.344680416979827}
{"item_id":"senseval2.d002.s031.t003","gold_sense_keys":["well%4:02:00::"],"candidates":[{"index":1,"sense_key":"well%4:02:07::","synset_id":"well.r.06"},{"index":2,"sense_key":"well%4:02:12::","synset_id":"well.r.10"},{"index":3,"sense_key":"well%4:02:03::","synset_id":"well.r.03"},{"index":4,"sense_key":"well%4:02:00::","synset_id":"well.r.01"},{"index":5,"sense_key":"well%4:02:13::","synset_id":"well.r.09"},{"index":6,"sense_key":"well%4:02:09::","synset_id":"well.r.12"},{"index":7,"sense_key":"well%4:02:05::","synset_id":"well.r.02"},{"index":8,"sense_key":"well%4:02:08::","synset_id":"well.r.05"},{"index":9,"sense_key":"well%4:02:01::","synset_id":"well.r.13"},{"index":10,"sense_key":"well%4:02:15::","synset_id":"well.r.04"},{"index":11,"sense_key":"well%4:02:10::","synset_id":"well.r.11"},{"index":12,"sense_key":"well%4:02:11::","synset_id":"well.r.07"},{"index":13,"sense_key":"well%4:02:14::","synset_id":"well.r.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"well%4:02:00::","call_ids":["senseval2.d002.s031.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"well%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":823,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.035495,"input_uncached_usd":0.004115000000000001,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":18.985968332970515}
{"item_id":"senseval2.d002.s032.t000","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"},{"index":2,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s032.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":321,"cached_input_tokens":0,"output_tokens":41,"reasoning_output_tokens":25},"cost":{"currency":"USD","total_usd":0.002835,"input_uncached_usd":0.001605,"input_cached_usd":0.0,"output_usd":0.00123,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3748368340311572}
{"item_id":"senseval2.d002.s032.t001","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"},{"index":2,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"},{"index":3,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":4,"sense_key":"important%3:00:00::","synset_id":"important.a.01"},{"index":5,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"important%3:00:04::","call_ids":["senseval2.d002.s032.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"important%3:00:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":6226,"reasoning_output_tokens":6144},"cost":{"currency":"USD","total_usd":0.189205,"input_uncached_usd":0.002425,"input_cached_usd":0.0,"output_usd":0.18678,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":99.20962670794688}
{"item_id":"senseval2.d002.s032.t002","gold_sense_keys":["factor%1:09:00::","factor%1:11:00::"],"candidates":[{"index":1,"sense_key":"factor%1:08:00::","synset_id":"gene.n.01"},{"index":2,"sense_key":"factor%1:09:01::","synset_id":"factor.n.06"},{"index":3,"sense_key":"factor%1:23:01::","synset_id":"factor.n.05"},{"index":4,"sense_key":"factor%1:09:00::","synset_id":"component.n.01"},{"index":5,"sense_key":"factor%1:11:00::","synset_id":"factor.n.01"},{"index":6,"sense_key":"factor%1:18:00::","synset_id":"agent.n.04"},{"index":7,"sense_key":"factor%1:23:00::","synset_id":"divisor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"factor%1:11:00::","call_ids":["senseval2.d002.s032.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"factor%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.004685,"input_uncached_usd":0.0028250000000000003,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.727608666056767}
{"item_id":"senseval2.d002.s032.t003","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s032.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":324,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.00441,"input_uncached_usd":0.0016200000000000001,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8035080829868093}
{"item_id":"senseval2.d002.s032.t004","gold_sense_keys":["crisis%1:26:00::"],"candidates":[{"index":1,"sense_key":"crisis%1:26:00::","synset_id":"crisis.n.01"},{"index":2,"sense_key":"crisis%1:11:00::","synset_id":"crisis.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crisis%1:26:00::","call_ids":["senseval2.d002.s032.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crisis%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":337,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.004895,"input_uncached_usd":0.001685,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.157808457966894}
{"item_id":"senseval2.d002.s033.t000","gold_sense_keys":["problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":2,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"problem%1:10:00::","call_ids":["senseval2.d002.s033.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"problem%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":392,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":122},"cost":{"currency":"USD","total_usd":0.006100000000000001,"input_uncached_usd":0.0019600000000000004,"input_cached_usd":0.0,"output_usd":0.0041400000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.062915000016801}
{"item_id":"senseval2.d002.s033.t001","gold_sense_keys":["equity%1:07:00::"],"candidates":[{"index":1,"sense_key":"equity%1:21:00::","synset_id":"equity.n.02"},{"index":2,"sense_key":"equity%1:07:00::","synset_id":"fairness.n.01"},{"index":3,"sense_key":"equity%1:21:01::","synset_id":"equity.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"equity%1:07:00::","call_ids":["senseval2.d002.s033.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"equity%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":371,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.003325,"input_uncached_usd":0.001855,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.490711999940686}
{"item_id":"senseval2.d002.s033.t002","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"},{"index":2,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s033.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":335,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.003745,"input_uncached_usd":0.001675,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8752265829825774}
{"item_id":"senseval2.d002.s033.t003","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"},{"index":3,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:10:00::","call_ids":["senseval2.d002.s033.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":391,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017795,"input_uncached_usd":0.001955,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.698765625013039}
{"item_id":"senseval2.d002.s034.t002","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"},{"index":2,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s034.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":336,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00336,"input_uncached_usd":0.00168,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.488484791945666}
{"item_id":"senseval2.d002.s034.t003","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":2,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":3,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":4,"sense_key":"think%2:30:00::","synset_id":"think.v.13"},{"index":5,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":6,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":7,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":8,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":9,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":10,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":11,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":12,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":13,"sense_key":"think%2:31:08::","synset_id":"think.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval2.d002.s034.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":843,"cached_input_tokens":0,"output_tokens":2401,"reasoning_output_tokens":2360},"cost":{"currency":"USD","total_usd":0.076245,"input_uncached_usd":0.004215,"input_cached_usd":0.0,"output_usd":0.07203,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":38.713043999974616}
{"item_id":"senseval2.d002.s034.t004","gold_sense_keys":["educate%2:41:00::"],"candidates":[{"index":1,"sense_key":"educate%2:41:00::","synset_id":"educate.v.01"},{"index":2,"sense_key":"educate%2:31:00::","synset_id":"train.v.01"},{"index":3,"sense_key":"educate%2:41:01::","synset_id":"educate.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educate%2:41:00::","call_ids":["senseval2.d002.s034.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educate%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":412,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.0038900000000000002,"input_uncached_usd":0.00206,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.681419000029564}
{"item_id":"senseval2.d002.s034.t005","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":2,"sense_key":"child%1:18:07::","synset_id":"child.n.04"},{"index":3,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":4,"sense_key":"child%1:18:06::","synset_id":"child.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"child%1:18:00::","call_ids":["senseval2.d002.s034.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"child%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.004824999999999999,"input_uncached_usd":0.002245,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.889049999997951}
{"item_id":"senseval2.d002.s034.t006","gold_sense_keys":["better%4:02:00::"],"candidates":[{"index":1,"sense_key":"well%4:02:10::","synset_id":"well.r.11"},{"index":2,"sense_key":"well%4:02:12::","synset_id":"well.r.10"},{"index":3,"sense_key":"well%4:02:05::","synset_id":"well.r.02"},{"index":4,"sense_key":"well%4:02:03::","synset_id":"well.r.03"},{"index":5,"sense_key":"well%4:02:13::","synset_id":"well.r.09"},{"index":6,"sense_key":"well%4:02:11::","synset_id":"well.r.07"},{"index":7,"sense_key":"better%4:02:00::","synset_id":"better.r.01"},{"index":8,"sense_key":"better%4:02:01::","synset_id":"better.r.02"},{"index":9,"sense_key":"well%4:02:15::","synset_id":"well.r.04"},{"index":10,"sense_key":"well%4:02:08::","synset_id":"well.r.05"},{"index":11,"sense_key":"well%4:02:09::","synset_id":"well.r.12"},{"index":12,"sense_key":"well%4:02:14::","synset_id":"well.r.08"},{"index":13,"sense_key":"well%4:02:01::","synset_id":"well.r.13"},{"index":14,"sense_key":"well%4:02:07::","synset_id":"well.r.06"},{"index":15,"sense_key":"well%4:02:00::","synset_id":"well.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"better%4:02:00::","call_ids":["senseval2.d002.s034.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"better%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":982,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.00785,"input_uncached_usd":0.00491,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9607532090740278}
{"item_id":"senseval2.d002.s034.t008","gold_sense_keys":["get%2:40:00::"],"candidates":[{"index":1,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":2,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":3,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":4,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":5,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":6,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":7,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":8,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":9,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":10,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":11,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":12,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":13,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":14,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":15,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":16,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":17,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":18,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":19,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":20,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":21,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":22,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":23,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":24,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":27,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":28,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":29,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":30,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":31,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":32,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":33,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":34,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":35,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"},{"index":36,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":34,"chosen_sense_key":"get%2:40:00::","call_ids":["senseval2.d002.s034.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":34,"predicted_sense_key":"get%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1908,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02538,"input_uncached_usd":0.009540000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.070950500085019}
{"item_id":"senseval2.d002.s034.t010","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:14:00::","synset_id":"class.n.06"},{"index":2,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":3,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":4,"sense_key":"year%1:28:01::","synset_id":"year.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d002.s034.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.004065,"input_uncached_usd":0.0023550000000000003,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.61353387509007}
{"item_id":"senseval2.d002.s034.t011","gold_sense_keys":["more%4:02:01::"],"candidates":[{"index":1,"sense_key":"more%4:02:01::","synset_id":"more.r.02"},{"index":2,"sense_key":"more%4:02:00::","synset_id":"more.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%4:02:01::","call_ids":["senseval2.d002.s034.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":359,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":93},"cost":{"currency":"USD","total_usd":0.0050349999999999995,"input_uncached_usd":0.0017950000000000002,"input_cached_usd":0.0,"output_usd":0.0032400000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9740681670373306}
{"item_id":"senseval2.d002.s035.t002","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s035.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":348,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.00333,"input_uncached_usd":0.0017400000000000002,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4440682498971}
{"item_id":"senseval2.d002.s035.t003","gold_sense_keys":["union%1:14:01::"],"candidates":[{"index":1,"sense_key":"union%1:26:00::","synset_id":"union.n.04"},{"index":2,"sense_key":"union%1:14:01::","synset_id":"union.n.01"},{"index":3,"sense_key":"union%1:14:03::","synset_id":"union.n.08"},{"index":4,"sense_key":"union%1:14:02::","synset_id":"union.n.07"},{"index":5,"sense_key":"union%1:22:00::","synset_id":"union.n.06"},{"index":6,"sense_key":"union%1:11:00::","synset_id":"union.n.09"},{"index":7,"sense_key":"union%1:04:00::","synset_id":"union.n.11"},{"index":8,"sense_key":"union%1:15:00::","synset_id":"union.n.02"},{"index":9,"sense_key":"union%1:26:02::","synset_id":"marriage.n.01"},{"index":10,"sense_key":"union%1:04:02::","synset_id":"coupling.n.03"},{"index":11,"sense_key":"union%1:06:01::","synset_id":"union.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"union%1:14:01::","call_ids":["senseval2.d002.s035.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"union%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":829,"cached_input_tokens":0,"output_tokens":43,"reasoning_output_tokens":27},"cost":{"currency":"USD","total_usd":0.005435000000000001,"input_uncached_usd":0.004145,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3241757079958916}
{"item_id":"senseval2.d002.s035.t004","gold_sense_keys":["mind%2:32:00::"],"candidates":[{"index":1,"sense_key":"mind%2:31:00::","synset_id":"mind.v.06"},{"index":2,"sense_key":"mind%2:41:00::","synset_id":"heed.v.01"},{"index":3,"sense_key":"mind%2:31:03::","synset_id":"beware.v.01"},{"index":4,"sense_key":"mind%2:32:00::","synset_id":"mind.v.01"},{"index":5,"sense_key":"mind%2:31:02::","synset_id":"mind.v.02"},{"index":6,"sense_key":"mind%2:41:01::","synset_id":"take_care.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"mind%2:32:00::","call_ids":["senseval2.d002.s035.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"mind%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.005730000000000001,"input_uncached_usd":0.00252,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6868460410041735}
{"item_id":"senseval2.d002.s035.t005","gold_sense_keys":["spread%2:32:00::","spread%2:35:00::"],"candidates":[{"index":1,"sense_key":"spread%2:38:00::","synset_id":"spread.v.07"},{"index":2,"sense_key":"spread%2:32:00::","synset_id":"circulate.v.02"},{"index":3,"sense_key":"spread%2:42:00::","synset_id":"spread.v.03"},{"index":4,"sense_key":"spread%2:35:00::","synset_id":"spread.v.01"},{"index":5,"sense_key":"spread%2:35:03::","synset_id":"unfold.v.04"},{"index":6,"sense_key":"spread%2:32:02::","synset_id":"go_around.v.02"},{"index":7,"sense_key":"spread%2:35:13::","synset_id":"spread.v.10"},{"index":8,"sense_key":"spread%2:38:02::","synset_id":"diffuse.v.01"},{"index":9,"sense_key":"spread%2:35:07::","synset_id":"spread.v.02"},{"index":10,"sense_key":"spread%2:35:12::","synset_id":"spread.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"spread%2:32:00::","call_ids":["senseval2.d002.s035.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"spread%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":694,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.00605,"input_uncached_usd":0.0034700000000000004,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.943407249986194}
{"item_id":"senseval2.d002.s035.t007","gold_sense_keys":["narrow%5:00:00:limited:00"],"candidates":[{"index":1,"sense_key":"narrow%3:00:00::","synset_id":"narrow.a.01"},{"index":2,"sense_key":"narrow%5:00:00:limited:00","synset_id":"narrow.s.01"},{"index":3,"sense_key":"narrow%3:00:06::","synset_id":"narrow-minded.a.02"},{"index":4,"sense_key":"narrow%5:00:00:careful:00","synset_id":"minute.s.02"},{"index":5,"sense_key":"narrow%3:00:02::","synset_id":"narrow.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"narrow%5:00:00:limited:00","call_ids":["senseval2.d002.s035.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"narrow%5:00:00:limited:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":196,"reasoning_output_tokens":180},"cost":{"currency":"USD","total_usd":0.008245,"input_uncached_usd":0.0023650000000000003,"input_cached_usd":0.0,"output_usd":0.00588,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.491606708033942}
{"item_id":"senseval2.d002.s035.t008","gold_sense_keys":["purpose%1:09:00::"],"candidates":[{"index":1,"sense_key":"purpose%1:09:00::","synset_id":"purpose.n.01"},{"index":2,"sense_key":"purpose%1:07:01::","synset_id":"determination.n.02"},{"index":3,"sense_key":"purpose%1:07:00::","synset_id":"function.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"purpose%1:09:00::","call_ids":["senseval2.d002.s035.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"purpose%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.00383,"input_uncached_usd":0.0022400000000000002,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.982221166952513}
{"item_id":"senseval2.d002.s036.t003","gold_sense_keys":["politician%1:18:00::","politician%1:18:01::"],"candidates":[{"index":1,"sense_key":"politician%1:18:00::","synset_id":"politician.n.02"},{"index":2,"sense_key":"politician%1:18:01::","synset_id":"politician.n.01"},{"index":3,"sense_key":"politician%1:18:02::","synset_id":"politician.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"politician%1:18:00::","call_ids":["senseval2.d002.s036.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"politician%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":386,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":117},"cost":{"currency":"USD","total_usd":0.005920000000000001,"input_uncached_usd":0.00193,"input_cached_usd":0.0,"output_usd":0.0039900000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6142514999955893}
{"item_id":"senseval2.d002.s036.t004","gold_sense_keys":["find%2:31:10::"],"candidates":[{"index":1,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":2,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":3,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":4,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":5,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":6,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":7,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":8,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":9,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":10,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":11,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":12,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"},{"index":13,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":14,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":15,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":16,"sense_key":"find%2:38:10::","synset_id":"find.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"find%2:31:10::","call_ids":["senseval2.d002.s036.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"find%2:31:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1086,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":117},"cost":{"currency":"USD","total_usd":0.009420000000000001,"input_uncached_usd":0.005430000000000001,"input_cached_usd":0.0,"output_usd":0.0039900000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5166729170596227}
{"item_id":"senseval2.d002.s036.t006","gold_sense_keys":["help%2:41:00::"],"candidates":[{"index":1,"sense_key":"help%2:29:00::","synset_id":"help.v.02"},{"index":2,"sense_key":"help%2:42:08::","synset_id":"help_oneself.v.01"},{"index":3,"sense_key":"help%2:42:00::","synset_id":"help.v.03"},{"index":4,"sense_key":"help%2:41:00::","synset_id":"help.v.01"},{"index":5,"sense_key":"help%2:34:04::","synset_id":"avail.v.03"},{"index":6,"sense_key":"help%2:30:13::","synset_id":"help.v.08"},{"index":7,"sense_key":"help%2:41:12::","synset_id":"help.v.06"},{"index":8,"sense_key":"help%2:34:00::","synset_id":"serve.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"help%2:41:00::","call_ids":["senseval2.d002.s036.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"help%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018955,"input_uncached_usd":0.003115,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.477112208027393}
{"item_id":"senseval2.d002.s036.t007","gold_sense_keys":["strike%2:38:08::","strike%2:41:02::"],"candidates":[{"index":1,"sense_key":"strike%2:40:00::","synset_id":"fall_upon.v.01"},{"index":2,"sense_key":"strike%2:39:00::","synset_id":"hit.v.09"},{"index":3,"sense_key":"strike%2:35:02::","synset_id":"hit.v.05"},{"index":4,"sense_key":"strike%2:36:02::","synset_id":"mint.v.01"},{"index":5,"sense_key":"strike%2:41:02::","synset_id":"come_to.v.03"},{"index":6,"sense_key":"strike%2:35:10::","synset_id":"strickle.v.02"},{"index":7,"sense_key":"strike%2:41:00::","synset_id":"strike.v.07"},{"index":8,"sense_key":"strike%2:35:00::","synset_id":"strike.v.01"},{"index":9,"sense_key":"strike%2:35:08::","synset_id":"strike.v.14"},{"index":10,"sense_key":"strike%2:38:02::","synset_id":"hit.v.12"},{"index":11,"sense_key":"strike%2:42:00::","synset_id":"fall.v.08"},{"index":12,"sense_key":"strike%2:32:00::","synset_id":"strike.v.05"},{"index":13,"sense_key":"strike%2:35:01::","synset_id":"hit.v.02"},{"index":14,"sense_key":"strike%2:38:08::","synset_id":"assume.v.05"},{"index":15,"sense_key":"strike%2:35:03::","synset_id":"strike.v.10"},{"index":16,"sense_key":"strike%2:33:00::","synset_id":"strike.v.04"},{"index":17,"sense_key":"strike%2:30:00::","synset_id":"strike.v.11"},{"index":18,"sense_key":"strike%2:31:00::","synset_id":"strike.v.21"},{"index":19,"sense_key":"strike%2:37:00::","synset_id":"affect.v.05"},{"index":20,"sense_key":"strike%2:35:09::","synset_id":"strike.v.20"},{"index":21,"sense_key":"strike%2:36:00::","synset_id":"strike.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"strike%2:38:08::","call_ids":["senseval2.d002.s036.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"strike%2:38:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1257,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.009375000000000001,"input_uncached_usd":0.006285000000000001,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7640342499362305}
{"item_id":"senseval2.d002.s036.t008","gold_sense_keys":["friendly%5:00:00:amicable:00"],"candidates":[{"index":1,"sense_key":"friendly%3:00:03::","synset_id":"friendly.a.03"},{"index":2,"sense_key":"friendly%3:00:02::","synset_id":"friendly.a.04"},{"index":3,"sense_key":"friendly%5:00:00:amicable:00","synset_id":"friendly.s.01"},{"index":4,"sense_key":"friendly%3:00:01::","synset_id":"friendly.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"friendly%5:00:00:amicable:00","call_ids":["senseval2.d002.s036.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"friendly%5:00:00:amicable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":108},"cost":{"currency":"USD","total_usd":0.005975,"input_uncached_usd":0.002285,"input_cached_usd":0.0,"output_usd":0.00369,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3658014580141753}
{"item_id":"senseval2.d002.s036.t009","gold_sense_keys":["posture%1:09:00::"],"candidates":[{"index":1,"sense_key":"posture%1:07:01::","synset_id":"carriage.n.03"},{"index":2,"sense_key":"posture%1:09:00::","synset_id":"position.n.12"},{"index":3,"sense_key":"posture%1:07:00::","synset_id":"position.n.04"},{"index":4,"sense_key":"posture%1:07:02::","synset_id":"military_capability.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"posture%1:09:00::","call_ids":["senseval2.d002.s036.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"posture%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":101},"cost":{"currency":"USD","total_usd":0.005855000000000001,"input_uncached_usd":0.0023450000000000003,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3508682909887284}
{"item_id":"senseval2.d002.s036.t010","gold_sense_keys":["behalf%1:04:00::"],"candidates":[{"index":1,"sense_key":"behalf%1:07:00::","synset_id":"behalf.n.02"},{"index":2,"sense_key":"behalf%1:04:00::","synset_id":"behalf.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"behalf%1:04:00::","call_ids":["senseval2.d002.s036.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"behalf%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":392,"cached_input_tokens":0,"output_tokens":791,"reasoning_output_tokens":769},"cost":{"currency":"USD","total_usd":0.02569,"input_uncached_usd":0.0019600000000000004,"input_cached_usd":0.0,"output_usd":0.02373,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.796176707954146}
{"item_id":"senseval2.d002.s036.t011","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":2,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"},{"index":3,"sense_key":"important%3:00:00::","synset_id":"important.a.01"},{"index":4,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"},{"index":5,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"important%3:00:00::","call_ids":["senseval2.d002.s036.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"important%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018435,"input_uncached_usd":0.002595,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.66000479098875}
{"item_id":"senseval2.d002.s037.t000","gold_sense_keys":["shred%1:23:00::"],"candidates":[{"index":1,"sense_key":"shred%1:06:00::","synset_id":"rag.n.01"},{"index":2,"sense_key":"shred%1:23:00::","synset_id":"shred.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"shred%1:23:00::","call_ids":["senseval2.d002.s037.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"shred%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":380,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.0034300000000000003,"input_uncached_usd":0.0019000000000000002,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2913008750183508}
{"item_id":"senseval2.d002.s037.t001","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":2,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"},{"index":3,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval2.d002.s037.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":421,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.005585,"input_uncached_usd":0.002105,"input_cached_usd":0.0,"output_usd":0.00348,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8971187920542434}
{"item_id":"senseval2.d002.s037.t002","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":2,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":3,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":4,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d002.s037.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":73},"cost":{"currency":"USD","total_usd":0.005005000000000001,"input_uncached_usd":0.0023650000000000003,"input_cached_usd":0.0,"output_usd":0.00264,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0924860410159454}
{"item_id":"senseval2.d002.s037.t003","gold_sense_keys":["thing%1:09:02::"],"candidates":[{"index":1,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":4,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":5,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":6,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":7,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":8,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"},{"index":9,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":10,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":11,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":12,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"thing%1:07:00::","call_ids":["senseval2.d002.s037.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"thing%1:07:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":696,"cached_input_tokens":0,"output_tokens":4574,"reasoning_output_tokens":4510},"cost":{"currency":"USD","total_usd":0.14070000000000002,"input_uncached_usd":0.0034800000000000005,"input_cached_usd":0.0,"output_usd":0.13722,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":66.17631595907733}
{"item_id":"senseval2.d002.s037.t004","gold_sense_keys":["equal%3:00:00::"],"candidates":[{"index":1,"sense_key":"equal%3:00:03::","synset_id":"adequate.a.01"},{"index":2,"sense_key":"equal%3:00:00::","synset_id":"equal.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"equal%3:00:00::","call_ids":["senseval2.d002.s037.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"equal%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":384,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.004110000000000001,"input_uncached_usd":0.0019200000000000003,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9114722500089556}
{"item_id":"senseval2.d002.s037.t006","gold_sense_keys":["differential%1:07:00::"],"candidates":[{"index":1,"sense_key":"differential%1:07:00::","synset_id":"differential.n.02"},{"index":2,"sense_key":"differential%1:09:00::","synset_id":"derived_function.n.01"},{"index":3,"sense_key":"differential%1:06:00::","synset_id":"differential_gear.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"differential%1:07:00::","call_ids":["senseval2.d002.s037.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"differential%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":429,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.0050550000000000005,"input_uncached_usd":0.002145,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0816558330552652}
{"item_id":"senseval2.d002.s037.t007","gold_sense_keys":["result%2:42:00::"],"candidates":[{"index":1,"sense_key":"result%2:42:00::","synset_id":"result.v.01"},{"index":2,"sense_key":"result%2:42:02::","synset_id":"leave.v.07"},{"index":3,"sense_key":"result%2:30:00::","synset_id":"result.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"result%2:42:00::","call_ids":["senseval2.d002.s037.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"result%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017920000000000002,"input_uncached_usd":0.0021100000000000003,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.05178962496575}
{"item_id":"senseval2.d002.s037.t008","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"},{"index":2,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s037.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":363,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.005025,"input_uncached_usd":0.0018150000000000002,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5683697500498965}
{"item_id":"senseval2.d002.s037.t009","gold_sense_keys":["differential%1:07:00::"],"candidates":[{"index":1,"sense_key":"differential%1:07:00::","synset_id":"differential.n.02"},{"index":2,"sense_key":"differential%1:06:00::","synset_id":"differential_gear.n.01"},{"index":3,"sense_key":"differential%1:09:00::","synset_id":"derived_function.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"differential%1:07:00::","call_ids":["senseval2.d002.s037.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"differential%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":86},"cost":{"currency":"USD","total_usd":0.00517,"input_uncached_usd":0.00214,"input_cached_usd":0.0,"output_usd":0.00303,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9521530829370022}
{"item_id":"senseval2.d002.s038.t001","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":2,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"},{"index":3,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval2.d002.s038.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":436,"cached_input_tokens":0,"output_tokens":189,"reasoning_output_tokens":174},"cost":{"currency":"USD","total_usd":0.00785,"input_uncached_usd":0.00218,"input_cached_usd":0.0,"output_usd":0.0056700000000000006,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.22198979195673}
{"item_id":"senseval2.d002.s038.t002","gold_sense_keys":["sure%3:00:00::"],"candidates":[{"index":1,"sense_key":"sure%5:00:00:secure:02","synset_id":"sure.s.02"},{"index":2,"sense_key":"sure%3:00:04::","synset_id":"certain.a.04"},{"index":3,"sense_key":"sure%3:00:00::","synset_id":"certain.a.02"},{"index":4,"sense_key":"sure%5:00:02:certain:01","synset_id":"indisputable.s.02"},{"index":5,"sense_key":"sure%5:00:00:steady:00","synset_id":"sure.s.06"},{"index":6,"sense_key":"sure%5:00:00:careful:00","synset_id":"certain.s.03"},{"index":7,"sense_key":"sure%5:00:00:trustworthy:00","synset_id":"sure.s.04"},{"index":8,"sense_key":"sure%5:00:00:true:00","synset_id":"sure.s.05"},{"index":9,"sense_key":"sure%5:00:00:reliable:00","synset_id":"certain.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"sure%3:00:00::","call_ids":["senseval2.d002.s038.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"sure%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.005785,"input_uncached_usd":0.0035050000000000003,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.555875958991237}
{"item_id":"senseval2.d002.s039.t000","gold_sense_keys":["wish%2:37:02::"],"candidates":[{"index":1,"sense_key":"wish%2:37:01::","synset_id":"wish.v.04"},{"index":2,"sense_key":"wish%2:37:02::","synset_id":"wish.v.02"},{"index":3,"sense_key":"wish%2:37:00::","synset_id":"wish.v.01"},{"index":4,"sense_key":"wish%2:32:02::","synset_id":"wish.v.03"},{"index":5,"sense_key":"wish%2:32:01::","synset_id":"wish.v.05"},{"index":6,"sense_key":"wish%2:32:00::","synset_id":"wish.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"wish%2:37:02::","call_ids":["senseval2.d002.s039.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"wish%2:37:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.006200000000000001,"input_uncached_usd":0.00266,"input_cached_usd":0.0,"output_usd":0.00354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2790506669552997}
{"item_id":"senseval2.d002.s039.t001","gold_sense_keys":["serious%3:00:00::"],"candidates":[{"index":1,"sense_key":"serious%3:00:02::","synset_id":"unplayful.a.01"},{"index":2,"sense_key":"serious%5:00:00:critical:03","synset_id":"dangerous.s.01"},{"index":3,"sense_key":"serious%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":4,"sense_key":"serious%5:00:00:important:00","synset_id":"serious.s.01"},{"index":5,"sense_key":"serious%5:00:00:difficult:00","synset_id":"serious.s.04"},{"index":6,"sense_key":"serious%3:00:00::","synset_id":"serious.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"serious%3:00:00::","call_ids":["senseval2.d002.s039.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"serious%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":581,"cached_input_tokens":0,"output_tokens":155,"reasoning_output_tokens":139},"cost":{"currency":"USD","total_usd":0.007555000000000001,"input_uncached_usd":0.002905,"input_cached_usd":0.0,"output_usd":0.0046500000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.191865999950096}
{"item_id":"senseval2.d002.s039.t002","gold_sense_keys":["american%3:01:01::"],"candidates":[{"index":1,"sense_key":"american%3:01:01::","synset_id":"american.a.01"},{"index":2,"sense_key":"american%3:01:00::","synset_id":"american.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%3:01:01::","call_ids":["senseval2.d002.s039.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":408,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00393,"input_uncached_usd":0.00204,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1029607499949634}
{"item_id":"senseval2.d002.s039.t003","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":2,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":3,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":4,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":5,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"},{"index":6,"sense_key":"education%1:09:00::","synset_id":"education.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s039.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":166,"reasoning_output_tokens":150},"cost":{"currency":"USD","total_usd":0.007935000000000001,"input_uncached_usd":0.002955,"input_cached_usd":0.0,"output_usd":0.00498,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6283896249951795}
{"item_id":"senseval2.d002.s039.t004","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":2,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":3,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":4,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":5,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":6,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":7,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":8,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":9,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":10,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s039.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":840,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020040000000000002,"input_uncached_usd":0.004200000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.031565791927278}
{"item_id":"senseval2.d002.s039.t005","gold_sense_keys":["exactly%4:02:01::"],"candidates":[{"index":1,"sense_key":"exactly%4:02:01::","synset_id":"precisely.r.01"},{"index":2,"sense_key":"exactly%4:02:04::","synset_id":"precisely.r.03"},{"index":3,"sense_key":"exactly%4:02:02::","synset_id":"precisely.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"exactly%4:02:01::","call_ids":["senseval2.d002.s039.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"exactly%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":138},"cost":{"currency":"USD","total_usd":0.0069900000000000006,"input_uncached_usd":0.0024000000000000002,"input_cached_usd":0.0,"output_usd":0.00459,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.179959582979791}
{"item_id":"senseval2.d002.s039.t006","gold_sense_keys":["just%4:02:05::"],"candidates":[{"index":1,"sense_key":"just%4:02:00::","synset_id":"merely.r.01"},{"index":2,"sense_key":"just%4:02:05::","synset_id":"precisely.r.01"},{"index":3,"sense_key":"just%4:02:04::","synset_id":"barely.r.01"},{"index":4,"sense_key":"just%4:02:01::","synset_id":"just.r.03"},{"index":5,"sense_key":"just%4:02:06::","synset_id":"just.r.06"},{"index":6,"sense_key":"just%4:02:03::","synset_id":"just.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"just%4:02:05::","call_ids":["senseval2.d002.s039.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"just%4:02:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":107},"cost":{"currency":"USD","total_usd":0.00676,"input_uncached_usd":0.0030700000000000002,"input_cached_usd":0.0,"output_usd":0.00369,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.166627416969277}
{"item_id":"senseval2.d002.s039.t007","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"},{"index":2,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"},{"index":3,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"},{"index":4,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":5,"sense_key":"important%3:00:00::","synset_id":"important.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"important%3:00:00::","call_ids":["senseval2.d002.s039.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"important%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":960,"reasoning_output_tokens":938},"cost":{"currency":"USD","total_usd":0.031545000000000004,"input_uncached_usd":0.002745,"input_cached_usd":0.0,"output_usd":0.0288,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":15.381954957963899}
{"item_id":"senseval2.d002.s040.t002","gold_sense_keys":["school%1:14:00::"],"candidates":[{"index":1,"sense_key":"school%1:14:01::","synset_id":"school.n.07"},{"index":2,"sense_key":"school%1:28:00::","synset_id":"school.n.05"},{"index":3,"sense_key":"school%1:14:00::","synset_id":"school.n.01"},{"index":4,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":5,"sense_key":"school%1:09:00::","synset_id":"school.n.03"},{"index":6,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":7,"sense_key":"school%1:06:00::","synset_id":"school.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"school%1:14:00::","call_ids":["senseval2.d002.s040.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"school%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.006285000000000001,"input_uncached_usd":0.003075,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.235913624987006}
{"item_id":"senseval2.d002.s040.t003","gold_sense_keys":["scattered%5:00:00:distributed:00"],"candidates":[{"index":1,"sense_key":"scattered%5:00:00:incoherent:00","synset_id":"confused.s.02"},{"index":2,"sense_key":"scattered%5:00:00:distributed:00","synset_id":"scattered.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"scattered%5:00:00:distributed:00","call_ids":["senseval2.d002.s040.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"scattered%5:00:00:distributed:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.0037100000000000006,"input_uncached_usd":0.0021200000000000004,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7259375839494169}
{"item_id":"senseval2.d002.s040.t005","gold_sense_keys":["poor%3:00:00::","poor%3:00:03::"],"candidates":[{"index":1,"sense_key":"poor%3:00:00::","synset_id":"poor.a.02"},{"index":2,"sense_key":"poor%5:00:00:unfortunate:00","synset_id":"hapless.s.01"},{"index":3,"sense_key":"poor%5:00:00:bad:00","synset_id":"poor.s.03"},{"index":4,"sense_key":"poor%3:00:02::","synset_id":"poor.a.04"},{"index":5,"sense_key":"poor%3:00:03::","synset_id":"poor.a.03"},{"index":6,"sense_key":"poor%5:00:00:insufficient:00","synset_id":"inadequate.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"poor%3:00:03::","call_ids":["senseval2.d002.s040.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"poor%3:00:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":136},"cost":{"currency":"USD","total_usd":0.007490000000000001,"input_uncached_usd":0.0029300000000000003,"input_cached_usd":0.0,"output_usd":0.00456,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.045914874994196}
{"item_id":"senseval2.d002.s040.t006","gold_sense_keys":["same%3:00:00::"],"candidates":[{"index":1,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":2,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":3,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"},{"index":4,"sense_key":"same%3:00:04::","synset_id":"like.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"same%3:00:02::","call_ids":["senseval2.d002.s040.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"same%3:00:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1023},"cost":{"currency":"USD","total_usd":0.033785,"input_uncached_usd":0.002405,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":17.07422545796726}
{"item_id":"senseval2.d002.s041.t001","gold_sense_keys":["majority%1:07:00::"],"candidates":[{"index":1,"sense_key":"majority%1:28:00::","synset_id":"majority.n.03"},{"index":2,"sense_key":"majority%1:23:00::","synset_id":"majority.n.02"},{"index":3,"sense_key":"majority%1:07:00::","synset_id":"majority.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"majority%1:07:00::","call_ids":["senseval2.d002.s041.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"majority%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.003705,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3199383330065757}
{"item_id":"senseval2.d002.s041.t003","gold_sense_keys":["school%1:14:00::"],"candidates":[{"index":1,"sense_key":"school%1:28:00::","synset_id":"school.n.05"},{"index":2,"sense_key":"school%1:09:00::","synset_id":"school.n.03"},{"index":3,"sense_key":"school%1:14:01::","synset_id":"school.n.07"},{"index":4,"sense_key":"school%1:14:00::","synset_id":"school.n.01"},{"index":5,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":6,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":7,"sense_key":"school%1:06:00::","synset_id":"school.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"school%1:14:00::","call_ids":["senseval2.d002.s041.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"school%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":253,"reasoning_output_tokens":238},"cost":{"currency":"USD","total_usd":0.010660000000000001,"input_uncached_usd":0.0030700000000000002,"input_cached_usd":0.0,"output_usd":0.00759,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.703731165966019}
{"item_id":"senseval2.d002.s041.t004","gold_sense_keys":["know%2:31:01::","know%2:31:02::","know%2:31:12::"],"candidates":[{"index":1,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":4,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":5,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":6,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":7,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":8,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":9,"sense_key":"know%2:31:14::","synset_id":"know.v.11"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:02::","synset_id":"know.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s041.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":198,"reasoning_output_tokens":183},"cost":{"currency":"USD","total_usd":0.010110000000000001,"input_uncached_usd":0.00417,"input_cached_usd":0.0,"output_usd":0.00594,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4160304580582306}
{"item_id":"senseval2.d002.s041.t005","gold_sense_keys":["effort%1:04:00::"],"candidates":[{"index":1,"sense_key":"effort%1:04:00::","synset_id":"attempt.n.01"},{"index":2,"sense_key":"effort%1:04:01::","synset_id":"effort.n.02"},{"index":3,"sense_key":"effort%1:04:02::","synset_id":"feat.n.01"},{"index":4,"sense_key":"effort%1:04:03::","synset_id":"campaign.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"effort%1:04:00::","call_ids":["senseval2.d002.s041.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"effort%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":430,"reasoning_output_tokens":414},"cost":{"currency":"USD","total_usd":0.01558,"input_uncached_usd":0.00268,"input_cached_usd":0.0,"output_usd":0.0129,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.747099541011266}
{"item_id":"senseval2.d002.s041.t006","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"},{"index":2,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s041.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.0039250000000000005,"input_uncached_usd":0.0019150000000000003,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7820876250043511}
{"item_id":"senseval2.d002.s041.t007","gold_sense_keys":["reform%1:04:00::"],"candidates":[{"index":1,"sense_key":"reform%1:04:00::","synset_id":"reform.n.01"},{"index":2,"sense_key":"reform%1:04:01::","synset_id":"reform.n.02"},{"index":3,"sense_key":"reform%1:04:02::","synset_id":"reform.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"reform%1:04:01::","call_ids":["senseval2.d002.s041.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"reform%1:04:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":213,"reasoning_output_tokens":198},"cost":{"currency":"USD","total_usd":0.008595,"input_uncached_usd":0.0022050000000000004,"input_cached_usd":0.0,"output_usd":0.00639,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.0141858329297975}
{"item_id":"senseval2.d002.s041.t008","gold_sense_keys":["doomed%5:00:00:unfortunate:00"],"candidates":[{"index":1,"sense_key":"doomed%5:00:00:dead:01","synset_id":"doomed.s.01"},{"index":2,"sense_key":"doomed%5:00:00:certain:03","synset_id":"doomed.s.04"},{"index":3,"sense_key":"doomed%5:00:00:unfortunate:00","synset_id":"doomed.s.03"},{"index":4,"sense_key":"doomed%5:00:00:lost:02","synset_id":"cursed.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"doomed%5:00:00:unfortunate:00","call_ids":["senseval2.d002.s041.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"doomed%5:00:00:unfortunate:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":122},"cost":{"currency":"USD","total_usd":0.00677,"input_uncached_usd":0.00266,"input_cached_usd":0.0,"output_usd":0.00411,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.056875707930885}
{"item_id":"senseval2.d002.s042.t000","gold_sense_keys":["really%4:02:02::"],"candidates":[{"index":1,"sense_key":"really%4:02:02::","synset_id":"in_truth.r.01"},{"index":2,"sense_key":"really%4:02:00::","synset_id":"very.r.01"},{"index":3,"sense_key":"really%4:02:03::","synset_id":"truly.r.01"},{"index":4,"sense_key":"really%4:02:04::","synset_id":"actually.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"really%4:02:04::","call_ids":["senseval2.d002.s042.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"really%4:02:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01844,"input_uncached_usd":0.0026000000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.129511500010267}
{"item_id":"senseval2.d002.s042.t001","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":2,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":3,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":4,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":5,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":6,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":7,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":8,"sense_key":"know%2:31:14::","synset_id":"know.v.11"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":11,"sense_key":"know%2:31:03::","synset_id":"know.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s042.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":832,"cached_input_tokens":0,"output_tokens":314,"reasoning_output_tokens":298},"cost":{"currency":"USD","total_usd":0.01358,"input_uncached_usd":0.0041600000000000005,"input_cached_usd":0.0,"output_usd":0.00942,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.681419082917273}
{"item_id":"senseval2.d002.s042.t002","gold_sense_keys":["need%2:42:00::"],"candidates":[{"index":1,"sense_key":"need%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"need%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"need%2:34:01::","synset_id":"need.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"need%2:42:00::","call_ids":["senseval2.d002.s042.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"need%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":3117,"reasoning_output_tokens":3072},"cost":{"currency":"USD","total_usd":0.095725,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.09351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":54.99910870799795}
{"item_id":"senseval2.d002.s042.t003","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":2,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":3,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":4,"sense_key":"know%2:31:14::","synset_id":"know.v.11"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":7,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":8,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":9,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":10,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":11,"sense_key":"know%2:31:02::","synset_id":"know.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s042.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":831,"cached_input_tokens":0,"output_tokens":173,"reasoning_output_tokens":157},"cost":{"currency":"USD","total_usd":0.009345000000000001,"input_uncached_usd":0.004155000000000001,"input_cached_usd":0.0,"output_usd":0.00519,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.780510959099047}
{"item_id":"senseval2.d002.s042.t005","gold_sense_keys":["assimilate%2:31:00::"],"candidates":[{"index":1,"sense_key":"assimilate%2:30:00::","synset_id":"assimilate.v.02"},{"index":2,"sense_key":"assimilate%2:30:02::","synset_id":"assimilate.v.05"},{"index":3,"sense_key":"assimilate%2:30:01::","synset_id":"assimilate.v.03"},{"index":4,"sense_key":"assimilate%2:35:00::","synset_id":"assimilate.v.04"},{"index":5,"sense_key":"assimilate%2:31:00::","synset_id":"absorb.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"assimilate%2:31:00::","call_ids":["senseval2.d002.s042.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"assimilate%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.0045850000000000005,"input_uncached_usd":0.0025150000000000003,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.734583624987863}
{"item_id":"senseval2.d002.s043.t000","gold_sense_keys":["respect%1:09:00::"],"candidates":[{"index":1,"sense_key":"respect%1:12:00::","synset_id":"regard.n.06"},{"index":2,"sense_key":"respect%1:04:03::","synset_id":"deference.n.01"},{"index":3,"sense_key":"respect%1:09:00::","synset_id":"respect.n.01"},{"index":4,"sense_key":"respect%1:09:01::","synset_id":"respect.n.03"},{"index":5,"sense_key":"respect%1:07:00::","synset_id":"deference.n.02"},{"index":6,"sense_key":"respect%1:26:00::","synset_id":"esteem.n.01"},{"index":7,"sense_key":"respect%1:04:02::","synset_id":"obedience.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"respect%1:09:00::","call_ids":["senseval2.d002.s043.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"respect%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.004805,"input_uncached_usd":0.0032150000000000004,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7236174159916118}
{"item_id":"senseval2.d002.s043.t005","gold_sense_keys":["rather%4:02:02::"],"candidates":[{"index":1,"sense_key":"rather%4:02:04::","synset_id":"rather.r.02"},{"index":2,"sense_key":"rather%4:02:01::","synset_id":"preferably.r.01"},{"index":3,"sense_key":"rather%4:02:00::","synset_id":"quite.r.01"},{"index":4,"sense_key":"rather%4:02:02::","synset_id":"rather.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"rather%4:02:02::","call_ids":["senseval2.d002.s043.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"rather%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.00549,"input_uncached_usd":0.00252,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.691003749961965}
{"item_id":"senseval2.d002.s043.t007","gold_sense_keys":["concerned%3:00:00::","concerned%5:00:00:involved:00"],"candidates":[{"index":1,"sense_key":"concerned%5:00:00:involved:00","synset_id":"concerned.s.01"},{"index":2,"sense_key":"concerned%3:00:00::","synset_id":"concerned.a.01"},{"index":3,"sense_key":"concerned%5:00:01:involved:00","synset_id":"implicated.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"concerned%3:00:00::","call_ids":["senseval2.d002.s043.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"concerned%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":468,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.00465,"input_uncached_usd":0.00234,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9203241659561172}
{"item_id":"senseval2.d002.s044.t001","gold_sense_keys":["inclined%3:00:02::"],"candidates":[{"index":1,"sense_key":"inclined%3:00:01::","synset_id":"inclined.a.02"},{"index":2,"sense_key":"inclined%5:00:00:willing:00","synset_id":"disposed.s.01"},{"index":3,"sense_key":"inclined%3:00:02::","synset_id":"inclined.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"inclined%3:00:02::","call_ids":["senseval2.d002.s044.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"inclined%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00419,"input_uncached_usd":0.0022700000000000003,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8899416249478236}
{"item_id":"senseval2.d002.s044.t002","gold_sense_keys":["echo%2:32:00::"],"candidates":[{"index":1,"sense_key":"echo%2:32:00::","synset_id":"repeat.v.04"},{"index":2,"sense_key":"echo%2:42:00::","synset_id":"echo.v.03"},{"index":3,"sense_key":"echo%2:39:00::","synset_id":"resound.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"echo%2:32:00::","call_ids":["senseval2.d002.s044.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"echo%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.004115,"input_uncached_usd":0.002225,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6592527499888092}
{"item_id":"senseval2.d002.s044.t003","gold_sense_keys":["conventional%3:00:00::","conventional%3:00:01::","conventional%5:00:00:orthodox:00"],"candidates":[{"index":1,"sense_key":"conventional%5:00:00:orthodox:00","synset_id":"conventional.s.01"},{"index":2,"sense_key":"conventional%5:00:00:formal:01","synset_id":"ceremonious.s.01"},{"index":3,"sense_key":"conventional%3:00:02::","synset_id":"conventional.a.03"},{"index":4,"sense_key":"conventional%5:00:00:traditional:00","synset_id":"conventional.s.03"},{"index":5,"sense_key":"conventional%3:00:01::","synset_id":"conventional.a.04"},{"index":6,"sense_key":"conventional%3:00:00::","synset_id":"conventional.a.01"},{"index":7,"sense_key":"conventional%5:00:00:nonrepresentational:00","synset_id":"conventional.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"conventional%3:00:01::","call_ids":["senseval2.d002.s044.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"conventional%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018910000000000003,"input_uncached_usd":0.0031000000000000003,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.08995774993673}
{"item_id":"senseval2.d002.s044.t004","gold_sense_keys":["pap%1:10:00::"],"candidates":[{"index":1,"sense_key":"pap%1:08:00::","synset_id":"nipple.n.01"},{"index":2,"sense_key":"pap%1:10:00::","synset_id":"pap.n.01"},{"index":3,"sense_key":"pap%1:13:00::","synset_id":"soft_diet.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pap%1:10:00::","call_ids":["senseval2.d002.s044.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pap%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.004045,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3780548330396414}
{"item_id":"senseval2.d002.s045.t000","gold_sense_keys":["thus%4:02:00::"],"candidates":[{"index":1,"sense_key":"thus%4:02:00::","synset_id":"therefore.r.01"},{"index":2,"sense_key":"thus%4:02:01::","synset_id":"thus.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"thus%4:02:00::","call_ids":["senseval2.d002.s045.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"thus%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":85},"cost":{"currency":"USD","total_usd":0.005185,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0315680000931025}
{"item_id":"senseval2.d002.s045.t001","gold_sense_keys":["recent%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"recent%5:00:00:past:00","synset_id":"late.s.02"},{"index":2,"sense_key":"recent%5:00:00:new:00","synset_id":"recent.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recent%5:00:00:past:00","call_ids":["senseval2.d002.s045.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recent%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":398,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.00406,"input_uncached_usd":0.00199,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5061512920074165}
{"item_id":"senseval2.d002.s045.t002","gold_sense_keys":["governor%1:18:00::"],"candidates":[{"index":1,"sense_key":"governor%1:06:00::","synset_id":"governor.n.02"},{"index":2,"sense_key":"governor%1:18:00::","synset_id":"governor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"governor%1:18:00::","call_ids":["senseval2.d002.s045.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"governor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00387,"input_uncached_usd":0.0019500000000000001,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8333695409819484}
{"item_id":"senseval2.d002.s045.t003","gold_sense_keys":["conference%1:10:00::","conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":2,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"},{"index":3,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"conference%1:14:00::","call_ids":["senseval2.d002.s045.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.0038900000000000002,"input_uncached_usd":0.0021200000000000004,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7780479589710012}
{"item_id":"senseval2.d002.s045.t004","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"},{"index":3,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:09:01::","synset_id":"education.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s045.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":126},"cost":{"currency":"USD","total_usd":0.0072,"input_uncached_usd":0.0029400000000000003,"input_cached_usd":0.0,"output_usd":0.00426,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.909809250035323}
{"item_id":"senseval2.d002.s045.t005","gold_sense_keys":["announce%2:32:02::"],"candidates":[{"index":1,"sense_key":"announce%2:32:07::","synset_id":"announce.v.03"},{"index":2,"sense_key":"announce%2:32:00::","synset_id":"announce.v.01"},{"index":3,"sense_key":"announce%2:32:01::","synset_id":"announce.v.04"},{"index":4,"sense_key":"announce%2:32:02::","synset_id":"announce.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"announce%2:32:02::","call_ids":["senseval2.d002.s045.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"announce%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01815,"input_uncached_usd":0.00231,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.36142966710031}
{"item_id":"senseval2.d002.s045.t006","gold_sense_keys":["country%1:14:00::","country%1:14:01::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":3,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["senseval2.d002.s045.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":571,"cached_input_tokens":0,"output_tokens":285,"reasoning_output_tokens":269},"cost":{"currency":"USD","total_usd":0.011405,"input_uncached_usd":0.0028550000000000003,"input_cached_usd":0.0,"output_usd":0.00855,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.36618474998977}
{"item_id":"senseval2.d002.s045.t007","gold_sense_keys":["need%2:34:00::"],"candidates":[{"index":1,"sense_key":"need%2:34:01::","synset_id":"need.v.03"},{"index":2,"sense_key":"need%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"need%2:42:00::","synset_id":"necessitate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"need%2:34:00::","call_ids":["senseval2.d002.s045.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"need%2:34:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":1564,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.04916,"input_uncached_usd":0.0022400000000000002,"input_cached_usd":0.0,"output_usd":0.04692,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":25.11751016706694}
{"item_id":"senseval2.d002.s045.t009","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"},{"index":2,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":3,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval2.d002.s045.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":361,"reasoning_output_tokens":345},"cost":{"currency":"USD","total_usd":0.01309,"input_uncached_usd":0.0022600000000000003,"input_cached_usd":0.0,"output_usd":0.010830000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.319321832968853}
{"item_id":"senseval2.d002.s045.t010","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:07::","synset_id":"child.n.04"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":4,"sense_key":"child%1:18:06::","synset_id":"child.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"child%1:18:00::","call_ids":["senseval2.d002.s045.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"child%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":73},"cost":{"currency":"USD","total_usd":0.00515,"input_uncached_usd":0.00248,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.362508542020805}
{"item_id":"senseval2.d002.s046.t001","gold_sense_keys":["development%1:04:01::","development%1:22:02::"],"candidates":[{"index":1,"sense_key":"development%1:10:00::","synset_id":"development.n.09"},{"index":2,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":3,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":4,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":5,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":6,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":7,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":8,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":9,"sense_key":"development%1:26:00::","synset_id":"development.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"development%1:22:02::","call_ids":["senseval2.d002.s046.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"development%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":792,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019799999999999998,"input_uncached_usd":0.00396,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.349867290933616}
{"item_id":"senseval2.d002.s046.t002","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":2,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval2.d002.s046.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":114},"cost":{"currency":"USD","total_usd":0.006125,"input_uncached_usd":0.002225,"input_cached_usd":0.0,"output_usd":0.0039000000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.512975165969692}
{"item_id":"senseval2.d002.s046.t003","gold_sense_keys":["governor%1:18:00::"],"candidates":[{"index":1,"sense_key":"governor%1:06:00::","synset_id":"governor.n.02"},{"index":2,"sense_key":"governor%1:18:00::","synset_id":"governor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"governor%1:18:00::","call_ids":["senseval2.d002.s046.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"governor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":384,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.00408,"input_uncached_usd":0.0019200000000000003,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1096736670006067}
{"item_id":"senseval2.d002.s046.t004","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":2,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":3,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"},{"index":4,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval2.d002.s046.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"seem%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":398,"reasoning_output_tokens":382},"cost":{"currency":"USD","total_usd":0.014435,"input_uncached_usd":0.0024950000000000003,"input_cached_usd":0.0,"output_usd":0.011940000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.128404791932553}
{"item_id":"senseval2.d002.s046.t005","gold_sense_keys":["more%4:02:00::"],"candidates":[{"index":1,"sense_key":"more%4:02:01::","synset_id":"more.r.02"},{"index":2,"sense_key":"more%4:02:00::","synset_id":"more.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"more%4:02:00::","call_ids":["senseval2.d002.s046.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"more%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.00422,"input_uncached_usd":0.002,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6151490000775084}
{"item_id":"senseval2.d002.s046.t007","gold_sense_keys":["need%1:17:00::","need%1:26:00::"],"candidates":[{"index":1,"sense_key":"need%1:26:00::","synset_id":"need.n.01"},{"index":2,"sense_key":"need%1:26:01::","synset_id":"indigence.n.01"},{"index":3,"sense_key":"need%1:03:00::","synset_id":"motivation.n.01"},{"index":4,"sense_key":"need%1:17:00::","synset_id":"need.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"need%1:17:00::","call_ids":["senseval2.d002.s046.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"need%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":410,"reasoning_output_tokens":394},"cost":{"currency":"USD","total_usd":0.014975,"input_uncached_usd":0.0026750000000000003,"input_cached_usd":0.0,"output_usd":0.0123,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.681762999971397}
{"item_id":"senseval2.d002.s047.t002","gold_sense_keys":["extend%2:30:01::"],"candidates":[{"index":1,"sense_key":"extend%2:30:09::","synset_id":"extend.v.09"},{"index":2,"sense_key":"extend%2:30:12::","synset_id":"extend.v.17"},{"index":3,"sense_key":"extend%2:30:01::","synset_id":"widen.v.04"},{"index":4,"sense_key":"extend%2:30:06::","synset_id":"carry.v.09"},{"index":5,"sense_key":"extend%2:40:05::","synset_id":"extend.v.04"},{"index":6,"sense_key":"extend%2:34:00::","synset_id":"strain.v.03"},{"index":7,"sense_key":"extend%2:29:00::","synset_id":"stretch.v.02"},{"index":8,"sense_key":"extend%2:30:02::","synset_id":"prolong.v.01"},{"index":9,"sense_key":"extend%2:35:10::","synset_id":"extend.v.13"},{"index":10,"sense_key":"extend%2:38:14::","synset_id":"gallop.v.03"},{"index":11,"sense_key":"extend%2:29:01::","synset_id":"exsert.v.01"},{"index":12,"sense_key":"extend%2:42:02::","synset_id":"extend.v.06"},{"index":13,"sense_key":"extend%2:30:15::","synset_id":"extend.v.15"},{"index":14,"sense_key":"extend%2:42:01::","synset_id":"cover.v.03"},{"index":15,"sense_key":"extend%2:40:04::","synset_id":"offer.v.05"},{"index":16,"sense_key":"extend%2:38:10::","synset_id":"unfold.v.03"},{"index":17,"sense_key":"extend%2:42:03::","synset_id":"run.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"extend%2:30:01::","call_ids":["senseval2.d002.s047.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"extend%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1087,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":112},"cost":{"currency":"USD","total_usd":0.009275,"input_uncached_usd":0.005435000000000001,"input_cached_usd":0.0,"output_usd":0.00384,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3613280840218067}
{"item_id":"senseval2.d002.s047.t003","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s047.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":373,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.004985,"input_uncached_usd":0.0018650000000000001,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5958630830282345}
{"item_id":"senseval2.d002.s047.t004","gold_sense_keys":["system%1:14:00::"],"candidates":[{"index":1,"sense_key":"system%1:14:00::","synset_id":"system.n.02"},{"index":2,"sense_key":"system%1:07:00::","synset_id":"organization.n.05"},{"index":3,"sense_key":"system%1:09:02::","synset_id":"arrangement.n.03"},{"index":4,"sense_key":"system%1:09:01::","synset_id":"system.n.07"},{"index":5,"sense_key":"system%1:08:00::","synset_id":"system.n.06"},{"index":6,"sense_key":"system%1:27:00::","synset_id":"system.n.03"},{"index":7,"sense_key":"system%1:08:01::","synset_id":"system.n.08"},{"index":8,"sense_key":"system%1:06:00::","synset_id":"system.n.01"},{"index":9,"sense_key":"system%1:09:00::","synset_id":"system.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"system%1:14:00::","call_ids":["senseval2.d002.s047.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"system%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":525,"reasoning_output_tokens":510},"cost":{"currency":"USD","total_usd":0.01956,"input_uncached_usd":0.0038100000000000005,"input_cached_usd":0.0,"output_usd":0.01575,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.623294166987762}
{"item_id":"senseval2.d002.s048.t000","gold_sense_keys":["desirable%3:00:00::"],"candidates":[{"index":1,"sense_key":"desirable%3:00:00::","synset_id":"desirable.a.01"},{"index":2,"sense_key":"desirable%5:00:00:eligible:00","synset_id":"desirable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"desirable%3:00:00::","call_ids":["senseval2.d002.s048.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"desirable%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":394,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00377,"input_uncached_usd":0.00197,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6203735410235822}
{"item_id":"senseval2.d002.s048.t002","gold_sense_keys":["program%1:09:01::"],"candidates":[{"index":1,"sense_key":"program%1:04:00::","synset_id":"program.n.08"},{"index":2,"sense_key":"program%1:09:00::","synset_id":"plan.n.01"},{"index":3,"sense_key":"program%1:10:00::","synset_id":"program.n.05"},{"index":4,"sense_key":"program%1:10:04::","synset_id":"platform.n.02"},{"index":5,"sense_key":"program%1:10:02::","synset_id":"program.n.07"},{"index":6,"sense_key":"program%1:10:01::","synset_id":"broadcast.n.02"},{"index":7,"sense_key":"program%1:09:01::","synset_id":"program.n.02"},{"index":8,"sense_key":"program%1:10:05::","synset_id":"course_of_study.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"program%1:09:01::","call_ids":["senseval2.d002.s048.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"program%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":137},"cost":{"currency":"USD","total_usd":0.0081,"input_uncached_usd":0.00351,"input_cached_usd":0.0,"output_usd":0.00459,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.853024124982767}
{"item_id":"senseval2.d002.s048.t003","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"},{"index":2,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s048.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":375,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":120},"cost":{"currency":"USD","total_usd":0.005955,"input_uncached_usd":0.0018750000000000001,"input_cached_usd":0.0,"output_usd":0.00408,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1678402919787914}
{"item_id":"senseval2.d002.s048.t004","gold_sense_keys":["program%1:09:01::"],"candidates":[{"index":1,"sense_key":"program%1:10:05::","synset_id":"course_of_study.n.01"},{"index":2,"sense_key":"program%1:04:00::","synset_id":"program.n.08"},{"index":3,"sense_key":"program%1:10:04::","synset_id":"platform.n.02"},{"index":4,"sense_key":"program%1:10:02::","synset_id":"program.n.07"},{"index":5,"sense_key":"program%1:09:01::","synset_id":"program.n.02"},{"index":6,"sense_key":"program%1:09:00::","synset_id":"plan.n.01"},{"index":7,"sense_key":"program%1:10:00::","synset_id":"program.n.05"},{"index":8,"sense_key":"program%1:10:01::","synset_id":"broadcast.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"program%1:09:01::","call_ids":["senseval2.d002.s048.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"program%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01935,"input_uncached_usd":0.00351,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.818035374977626}
{"item_id":"senseval2.d002.s049.t000","gold_sense_keys":["know%2:31:01::","know%2:31:02::"],"candidates":[{"index":1,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":2,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":3,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":4,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":5,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":6,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":7,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":8,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":9,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":10,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s049.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":835,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020015,"input_uncached_usd":0.004175000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.068033625022508}
{"item_id":"senseval2.d002.s049.t001","gold_sense_keys":["very%4:02:00::"],"candidates":[{"index":1,"sense_key":"very%4:02:00::","synset_id":"very.r.01"},{"index":2,"sense_key":"very%4:02:01::","synset_id":"very.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"very%4:02:00::","call_ids":["senseval2.d002.s049.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"very%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.005294999999999999,"input_uncached_usd":0.002085,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1495156249729916}
{"item_id":"senseval2.d002.s049.t002","gold_sense_keys":["early%3:00:00::","early%3:00:02::"],"candidates":[{"index":1,"sense_key":"early%3:00:02::","synset_id":"early.a.02"},{"index":2,"sense_key":"early%3:00:00::","synset_id":"early.a.01"},{"index":3,"sense_key":"early%5:00:00:young:00","synset_id":"early.s.02"},{"index":4,"sense_key":"early%5:00:00:past:00","synset_id":"early.s.01"},{"index":5,"sense_key":"early%5:00:01:future:00","synset_id":"early.s.03"},{"index":6,"sense_key":"early%3:00:01::","synset_id":"early.a.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"early%3:00:00::","call_ids":["senseval2.d002.s049.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"early%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01876,"input_uncached_usd":0.0029200000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.181136875064112}
{"item_id":"senseval2.d002.s049.t003","gold_sense_keys":["exposure%1:04:06::"],"candidates":[{"index":1,"sense_key":"exposure%1:04:01::","synset_id":"exposure.n.10"},{"index":2,"sense_key":"exposure%1:06:00::","synset_id":"photograph.n.01"},{"index":3,"sense_key":"exposure%1:23:00::","synset_id":"exposure.n.06"},{"index":4,"sense_key":"exposure%1:04:05::","synset_id":"exposure.n.09"},{"index":5,"sense_key":"exposure%1:04:02::","synset_id":"exposure.n.08"},{"index":6,"sense_key":"exposure%1:09:00::","synset_id":"exposure.n.04"},{"index":7,"sense_key":"exposure%1:26:01::","synset_id":"vulnerability.n.01"},{"index":8,"sense_key":"exposure%1:07:00::","synset_id":"exposure.n.01"},{"index":9,"sense_key":"exposure%1:10:00::","synset_id":"exposure.n.03"},{"index":10,"sense_key":"exposure%1:04:06::","synset_id":"exposure.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"exposure%1:04:06::","call_ids":["senseval2.d002.s049.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"exposure%1:04:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":709,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.005495,"input_uncached_usd":0.0035450000000000004,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6953001669608057}
{"item_id":"senseval2.d002.s049.t004","gold_sense_keys":["schooling%1:09:00::"],"candidates":[{"index":1,"sense_key":"schooling%1:09:00::","synset_id":"school.n.03"},{"index":2,"sense_key":"schooling%1:04:01::","synset_id":"schooling.n.03"},{"index":3,"sense_key":"schooling%1:04:00::","synset_id":"schooling.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"schooling%1:09:00::","call_ids":["senseval2.d002.s049.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"schooling%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":124},"cost":{"currency":"USD","total_usd":0.006300000000000001,"input_uncached_usd":0.0021000000000000003,"input_cached_usd":0.0,"output_usd":0.0042,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.195202250033617}
{"item_id":"senseval2.d002.s049.t005","gold_sense_keys":["improve%2:30:01::"],"candidates":[{"index":1,"sense_key":"improve%2:30:01::","synset_id":"better.v.02"},{"index":2,"sense_key":"improve%2:30:00::","synset_id":"better.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"improve%2:30:01::","call_ids":["senseval2.d002.s049.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"improve%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.004825,"input_uncached_usd":0.0020350000000000004,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.784455250017345}
{"item_id":"senseval2.d002.s049.t006","gold_sense_keys":["performance%1:04:00::","performance%1:04:03::"],"candidates":[{"index":1,"sense_key":"performance%1:10:00::","synset_id":"performance.n.01"},{"index":2,"sense_key":"performance%1:04:01::","synset_id":"performance.n.02"},{"index":3,"sense_key":"performance%1:22:00::","synset_id":"operation.n.08"},{"index":4,"sense_key":"performance%1:04:00::","synset_id":"performance.n.03"},{"index":5,"sense_key":"performance%1:04:03::","synset_id":"performance.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"performance%1:04:00::","call_ids":["senseval2.d002.s049.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"performance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":571,"cached_input_tokens":0,"output_tokens":159,"reasoning_output_tokens":143},"cost":{"currency":"USD","total_usd":0.007625000000000002,"input_uncached_usd":0.0028550000000000003,"input_cached_usd":0.0,"output_usd":0.00477,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.01725879102014}
{"item_id":"senseval2.d002.s049.t007","gold_sense_keys":["first%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"first%3:00:00::","synset_id":"first.a.01"},{"index":2,"sense_key":"first%3:00:03::","synset_id":"first.a.06"},{"index":3,"sense_key":"first%5:00:01:opening:00","synset_id":"beginning.s.01"},{"index":4,"sense_key":"first%5:00:00:best:00","synset_id":"first.s.04"},{"index":5,"sense_key":"first%5:00:00:ordinal:00","synset_id":"first.s.01"},{"index":6,"sense_key":"first%5:00:02:opening:00","synset_id":"inaugural.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"first%5:00:00:ordinal:00","call_ids":["senseval2.d002.s049.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"first%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.006435000000000001,"input_uncached_usd":0.0028950000000000004,"input_cached_usd":0.0,"output_usd":0.00354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.316918749944307}
{"item_id":"senseval2.d002.s049.t008","gold_sense_keys":["grade%1:14:00::"],"candidates":[{"index":1,"sense_key":"grade%1:07:01::","synset_id":"grade.n.03"},{"index":2,"sense_key":"grade%1:26:00::","synset_id":"grade.n.02"},{"index":3,"sense_key":"grade%1:07:02::","synset_id":"grade.n.07"},{"index":4,"sense_key":"grade%1:09:00::","synset_id":"mark.n.01"},{"index":5,"sense_key":"grade%1:05:00::","synset_id":"grade.n.09"},{"index":6,"sense_key":"grade%1:23:00::","synset_id":"grad.n.01"},{"index":7,"sense_key":"grade%1:14:00::","synset_id":"class.n.02"},{"index":8,"sense_key":"grade%1:07:00::","synset_id":"degree.n.01"},{"index":9,"sense_key":"grade%1:10:00::","synset_id":"grade.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"grade%1:14:00::","call_ids":["senseval2.d002.s049.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"grade%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":496,"reasoning_output_tokens":480},"cost":{"currency":"USD","total_usd":0.01817,"input_uncached_usd":0.0032900000000000004,"input_cached_usd":0.0,"output_usd":0.01488,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.873401083983481}
{"item_id":"senseval2.d002.s049.t010","gold_sense_keys":["difference%1:07:00::"],"candidates":[{"index":1,"sense_key":"difference%1:10:00::","synset_id":"dispute.n.01"},{"index":2,"sense_key":"difference%1:24:00::","synset_id":"difference.n.04"},{"index":3,"sense_key":"difference%1:11:00::","synset_id":"deviation.n.01"},{"index":4,"sense_key":"difference%1:23:00::","synset_id":"remainder.n.03"},{"index":5,"sense_key":"difference%1:07:00::","synset_id":"difference.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"difference%1:24:00::","call_ids":["senseval2.d002.s049.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"difference%1:24:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018485,"input_uncached_usd":0.002645,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.127321625011973}
{"item_id":"senseval2.d002.s049.t011","gold_sense_keys":["quickly%4:02:02::"],"candidates":[{"index":1,"sense_key":"quickly%4:02:00::","synset_id":"quickly.r.01"},{"index":2,"sense_key":"quickly%4:02:02::","synset_id":"promptly.r.01"},{"index":3,"sense_key":"quickly%4:02:03::","synset_id":"cursorily.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"quickly%4:02:00::","call_ids":["senseval2.d002.s049.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"quickly%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":318,"reasoning_output_tokens":303},"cost":{"currency":"USD","total_usd":0.011750000000000002,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.00954,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.408847291953862}
{"item_id":"senseval2.d002.s049.t013","gold_sense_keys":["wash_away%2:30:01::"],"candidates":[{"index":1,"sense_key":"wash_away%2:30:00::","synset_id":"wash.v.09"},{"index":2,"sense_key":"wash_away%2:30:01::","synset_id":"wash_away.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"wash_away%2:30:01::","call_ids":["senseval2.d002.s049.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"wash_away%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":413,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.003835,"input_uncached_usd":0.002065,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6842574590118602}
{"item_id":"senseval2.d002.s050.t000","gold_sense_keys":["sum_up%2:32:00::"],"candidates":[{"index":1,"sense_key":"sum_up%2:42:00::","synset_id":"summarize.v.02"},{"index":2,"sense_key":"sum_up%2:32:03::","synset_id":"total.v.02"},{"index":3,"sense_key":"sum_up%2:32:00::","synset_id":"sum_up.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"sum_up%2:32:00::","call_ids":["senseval2.d002.s050.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"sum_up%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":458,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.00406,"input_uncached_usd":0.0022900000000000004,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6186921669868752}
{"item_id":"senseval2.d002.s050.t001","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":3,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":4,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":5,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":6,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":7,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":8,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":9,"sense_key":"know%2:31:14::","synset_id":"know.v.11"},{"index":10,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":11,"sense_key":"know%2:31:02::","synset_id":"know.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s050.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":501,"reasoning_output_tokens":486},"cost":{"currency":"USD","total_usd":0.019165,"input_uncached_usd":0.004135000000000001,"input_cached_usd":0.0,"output_usd":0.01503,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.33472091704607}
{"item_id":"senseval2.d002.s050.t002","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":4,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":5,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s050.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":81},"cost":{"currency":"USD","total_usd":0.005805,"input_uncached_usd":0.0028950000000000004,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.468661417020485}
{"item_id":"senseval2.d002.s050.t003","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s050.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":187,"reasoning_output_tokens":171},"cost":{"currency":"USD","total_usd":0.008505,"input_uncached_usd":0.0028950000000000004,"input_cached_usd":0.0,"output_usd":0.00561,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.426378833944909}
{"item_id":"senseval2.d002.s050.t004","gold_sense_keys":["reform%1:04:00::"],"candidates":[{"index":1,"sense_key":"reform%1:04:01::","synset_id":"reform.n.02"},{"index":2,"sense_key":"reform%1:04:02::","synset_id":"reform.n.03"},{"index":3,"sense_key":"reform%1:04:00::","synset_id":"reform.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reform%1:04:01::","call_ids":["senseval2.d002.s050.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reform%1:04:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018009999999999998,"input_uncached_usd":0.00217,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.71172550006304}
{"item_id":"senseval2.d002.s050.t005","gold_sense_keys":["work%2:41:03::"],"candidates":[{"index":1,"sense_key":"work%2:36:12::","synset_id":"shape.v.02"},{"index":2,"sense_key":"work%2:35:02::","synset_id":"function.v.01"},{"index":3,"sense_key":"work%2:41:10::","synset_id":"influence.v.01"},{"index":4,"sense_key":"work%2:36:00::","synset_id":"work.v.05"},{"index":5,"sense_key":"work%2:29:00::","synset_id":"exercise.v.03"},{"index":6,"sense_key":"work%2:41:03::","synset_id":"work.v.03"},{"index":7,"sense_key":"work%2:41:01::","synset_id":"work.v.12"},{"index":8,"sense_key":"work%2:35:00::","synset_id":"knead.v.01"},{"index":9,"sense_key":"work%2:38:02::","synset_id":"work.v.10"},{"index":10,"sense_key":"work%2:41:02::","synset_id":"work.v.01"},{"index":11,"sense_key":"work%2:41:04::","synset_id":"work.v.08"},{"index":12,"sense_key":"work%2:42:10::","synset_id":"work.v.14"},{"index":13,"sense_key":"work%2:41:00::","synset_id":"work.v.02"},{"index":14,"sense_key":"work%2:41:05::","synset_id":"work.v.09"},{"index":15,"sense_key":"work%2:31:13::","synset_id":"solve.v.01"},{"index":16,"sense_key":"work%2:37:10::","synset_id":"work.v.19"},{"index":17,"sense_key":"work%2:30:10::","synset_id":"work.v.27"},{"index":18,"sense_key":"work%2:41:11::","synset_id":"work.v.16"},{"index":19,"sense_key":"work%2:34:10::","synset_id":"exploit.v.01"},{"index":20,"sense_key":"work%2:36:09::","synset_id":"cultivate.v.02"},{"index":21,"sense_key":"work%2:38:00::","synset_id":"make.v.36"},{"index":22,"sense_key":"work%2:30:14::","synset_id":"ferment.v.03"},{"index":23,"sense_key":"work%2:39:10::","synset_id":"work.v.18"},{"index":24,"sense_key":"work%2:36:01::","synset_id":"bring.v.03"},{"index":25,"sense_key":"work%2:30:13::","synset_id":"sour.v.01"},{"index":26,"sense_key":"work%2:41:12::","synset_id":"work.v.17"},{"index":27,"sense_key":"work%2:35:14::","synset_id":"work.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"work%2:41:03::","call_ids":["senseval2.d002.s050.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"work%2:41:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1487,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":109},"cost":{"currency":"USD","total_usd":0.011185,"input_uncached_usd":0.007435000000000001,"input_cached_usd":0.0,"output_usd":0.0037500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4494839580729604}
{"item_id":"senseval2.d002.s050.t006","gold_sense_keys":["work%2:41:03::"],"candidates":[{"index":1,"sense_key":"work%2:37:10::","synset_id":"work.v.19"},{"index":2,"sense_key":"work%2:29:00::","synset_id":"exercise.v.03"},{"index":3,"sense_key":"work%2:38:00::","synset_id":"make.v.36"},{"index":4,"sense_key":"work%2:41:11::","synset_id":"work.v.16"},{"index":5,"sense_key":"work%2:30:10::","synset_id":"work.v.27"},{"index":6,"sense_key":"work%2:35:14::","synset_id":"work.v.21"},{"index":7,"sense_key":"work%2:41:12::","synset_id":"work.v.17"},{"index":8,"sense_key":"work%2:36:00::","synset_id":"work.v.05"},{"index":9,"sense_key":"work%2:30:13::","synset_id":"sour.v.01"},{"index":10,"sense_key":"work%2:31:13::","synset_id":"solve.v.01"},{"index":11,"sense_key":"work%2:41:04::","synset_id":"work.v.08"},{"index":12,"sense_key":"work%2:35:00::","synset_id":"knead.v.01"},{"index":13,"sense_key":"work%2:42:10::","synset_id":"work.v.14"},{"index":14,"sense_key":"work%2:30:14::","synset_id":"ferment.v.03"},{"index":15,"sense_key":"work%2:41:02::","synset_id":"work.v.01"},{"index":16,"sense_key":"work%2:41:10::","synset_id":"influence.v.01"},{"index":17,"sense_key":"work%2:41:00::","synset_id":"work.v.02"},{"index":18,"sense_key":"work%2:36:01::","synset_id":"bring.v.03"},{"index":19,"sense_key":"work%2:41:01::","synset_id":"work.v.12"},{"index":20,"sense_key":"work%2:35:02::","synset_id":"function.v.01"},{"index":21,"sense_key":"work%2:41:03::","synset_id":"work.v.03"},{"index":22,"sense_key":"work%2:36:12::","synset_id":"shape.v.02"},{"index":23,"sense_key":"work%2:34:10::","synset_id":"exploit.v.01"},{"index":24,"sense_key":"work%2:39:10::","synset_id":"work.v.18"},{"index":25,"sense_key":"work%2:41:05::","synset_id":"work.v.09"},{"index":26,"sense_key":"work%2:38:02::","synset_id":"work.v.10"},{"index":27,"sense_key":"work%2:36:09::","synset_id":"cultivate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":21,"chosen_sense_key":"work%2:41:03::","call_ids":["senseval2.d002.s050.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":21,"predicted_sense_key":"work%2:41:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1486,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":139},"cost":{"currency":"USD","total_usd":0.012050000000000002,"input_uncached_usd":0.007430000000000001,"input_cached_usd":0.0,"output_usd":0.00462,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.245982041931711}
{"item_id":"senseval2.d002.s050.t007","gold_sense_keys":["parental%3:00:00::"],"candidates":[{"index":1,"sense_key":"parental%3:01:00::","synset_id":"parental.a.01"},{"index":2,"sense_key":"parental%3:00:00::","synset_id":"parental.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"parental%3:00:00::","call_ids":["senseval2.d002.s050.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"parental%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":384,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.00351,"input_uncached_usd":0.0019200000000000003,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5439081250224262}
{"item_id":"senseval2.d002.s050.t008","gold_sense_keys":["involvement%1:04:00::"],"candidates":[{"index":1,"sense_key":"involvement%1:24:00::","synset_id":"involvement.n.02"},{"index":2,"sense_key":"involvement%1:04:00::","synset_id":"engagement.n.07"},{"index":3,"sense_key":"involvement%1:26:01::","synset_id":"participation.n.02"},{"index":4,"sense_key":"involvement%1:09:00::","synset_id":"interest.n.01"},{"index":5,"sense_key":"involvement%1:26:00::","synset_id":"affair.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"involvement%1:04:00::","call_ids":["senseval2.d002.s050.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"involvement%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.005595,"input_uncached_usd":0.002625,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.332067667040974}
{"item_id":"senseval2.d002.s050.t009","gold_sense_keys":["bad%3:00:00::"],"candidates":[{"index":1,"sense_key":"bad%5:00:00:inferior:02","synset_id":"bad.s.05"},{"index":2,"sense_key":"bad%3:00:00::","synset_id":"bad.a.01"},{"index":3,"sense_key":"bad%5:00:00:malfunctioning:00","synset_id":"bad.s.12"},{"index":4,"sense_key":"bad%5:00:00:stale:00","synset_id":"bad.s.03"},{"index":5,"sense_key":"bad%5:00:00:unhealthy:00","synset_id":"bad.s.08"},{"index":6,"sense_key":"bad%5:00:00:nonstandard:03","synset_id":"bad.s.06"},{"index":7,"sense_key":"bad%5:00:00:invalid:00","synset_id":"bad.s.04"},{"index":8,"sense_key":"bad%5:00:02:unsound:00","synset_id":"bad.s.07"},{"index":9,"sense_key":"bad%5:00:00:counterfeit:00","synset_id":"bad.s.11"},{"index":10,"sense_key":"bad%5:00:00:intense:00","synset_id":"bad.s.01"},{"index":11,"sense_key":"bad%3:00:09::","synset_id":"regretful.a.01"},{"index":12,"sense_key":"bad%5:00:00:evil:00","synset_id":"bad.s.10"},{"index":13,"sense_key":"bad%5:00:00:harmful:00","synset_id":"bad.s.09"},{"index":14,"sense_key":"bad%5:00:00:uncomfortable:00","synset_id":"bad.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bad%3:00:00::","call_ids":["senseval2.d002.s050.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bad%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":913,"cached_input_tokens":0,"output_tokens":172,"reasoning_output_tokens":156},"cost":{"currency":"USD","total_usd":0.009725000000000001,"input_uncached_usd":0.0045650000000000005,"input_cached_usd":0.0,"output_usd":0.0051600000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.656631749938242}
{"item_id":"senseval2.d002.s050.t010","gold_sense_keys":["idea%1:09:00::"],"candidates":[{"index":1,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"},{"index":2,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"},{"index":3,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"},{"index":4,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"},{"index":5,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"idea%1:09:00::","call_ids":["senseval2.d002.s050.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"idea%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":117},"cost":{"currency":"USD","total_usd":0.006765,"input_uncached_usd":0.002775,"input_cached_usd":0.0,"output_usd":0.0039900000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9336188749875873}
{"item_id":"senseval2.d002.s051.t000","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"},{"index":2,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s051.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":371,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.003265,"input_uncached_usd":0.001855,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8722283749375492}
{"item_id":"senseval2.d002.s051.t001","gold_sense_keys":["too%4:02:00::"],"candidates":[{"index":1,"sense_key":"too%4:02:00::","synset_id":"excessively.r.01"},{"index":2,"sense_key":"too%4:02:01::","synset_id":"besides.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"too%4:02:00::","call_ids":["senseval2.d002.s051.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"too%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":375,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.003615,"input_uncached_usd":0.0018750000000000001,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9302445830544457}
{"item_id":"senseval2.d002.s051.t002","gold_sense_keys":["likely%3:00:00::"],"candidates":[{"index":1,"sense_key":"likely%5:00:00:prospective:00","synset_id":"likely.s.01"},{"index":2,"sense_key":"likely%3:00:04::","synset_id":"probable.a.01"},{"index":3,"sense_key":"likely%5:00:00:credible:00","synset_id":"likely.s.02"},{"index":4,"sense_key":"likely%3:00:00::","synset_id":"likely.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"likely%3:00:00::","call_ids":["senseval2.d002.s051.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"likely%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":104},"cost":{"currency":"USD","total_usd":0.005900000000000001,"input_uncached_usd":0.0023000000000000004,"input_cached_usd":0.0,"output_usd":0.0036,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3908802500227466}
{"item_id":"senseval2.d002.s051.t003","gold_sense_keys":["blame%2:31:00::","blame%2:32:00::"],"candidates":[{"index":1,"sense_key":"blame%2:31:00::","synset_id":"blame.v.03"},{"index":2,"sense_key":"blame%2:32:00::","synset_id":"blame.v.01"},{"index":3,"sense_key":"blame%2:32:01::","synset_id":"blame.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"blame%2:32:00::","call_ids":["senseval2.d002.s051.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"blame%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":405,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017865,"input_uncached_usd":0.0020250000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.602447374956682}
{"item_id":"senseval2.d002.s051.t004","gold_sense_keys":["school%1:14:00::"],"candidates":[{"index":1,"sense_key":"school%1:28:00::","synset_id":"school.n.05"},{"index":2,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":3,"sense_key":"school%1:06:00::","synset_id":"school.n.02"},{"index":4,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":5,"sense_key":"school%1:09:00::","synset_id":"school.n.03"},{"index":6,"sense_key":"school%1:14:00::","synset_id":"school.n.01"},{"index":7,"sense_key":"school%1:14:01::","synset_id":"school.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"school%1:14:00::","call_ids":["senseval2.d002.s051.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"school%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":590,"cached_input_tokens":0,"output_tokens":472,"reasoning_output_tokens":456},"cost":{"currency":"USD","total_usd":0.01711,"input_uncached_usd":0.0029500000000000004,"input_cached_usd":0.0,"output_usd":0.01416,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.050554333021864}
{"item_id":"senseval2.d002.s051.t005","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"},{"index":2,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s051.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":358,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.00455,"input_uncached_usd":0.0017900000000000001,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0505841670092195}
{"item_id":"senseval2.d002.s051.t006","gold_sense_keys":["limitation%1:07:00::"],"candidates":[{"index":1,"sense_key":"limitation%1:09:00::","synset_id":"restriction.n.01"},{"index":2,"sense_key":"limitation%1:23:00::","synset_id":"limit.n.06"},{"index":3,"sense_key":"limitation%1:07:00::","synset_id":"limitation.n.02"},{"index":4,"sense_key":"limitation%1:04:00::","synset_id":"limitation.n.05"},{"index":5,"sense_key":"limitation%1:28:00::","synset_id":"limitation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"limitation%1:07:00::","call_ids":["senseval2.d002.s051.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"limitation%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":95},"cost":{"currency":"USD","total_usd":0.005815000000000001,"input_uncached_usd":0.0024850000000000002,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.161788958008401}
{"item_id":"senseval2.d002.s051.t007","gold_sense_keys":["child%1:18:01::"],"candidates":[{"index":1,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":2,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":3,"sense_key":"child%1:18:07::","synset_id":"child.n.04"},{"index":4,"sense_key":"child%1:18:00::","synset_id":"child.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d002.s051.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"child%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":270,"reasoning_output_tokens":254},"cost":{"currency":"USD","total_usd":0.010435,"input_uncached_usd":0.0023350000000000003,"input_cached_usd":0.0,"output_usd":0.0081,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.415065500070341}
{"item_id":"senseval2.d002.s052.t000","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"},{"index":2,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s052.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":388,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.00335,"input_uncached_usd":0.00194,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6727236669976264}
{"item_id":"senseval2.d002.s052.t001","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"},{"index":2,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"},{"index":3,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"},{"index":4,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"},{"index":5,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"involved%3:00:00::","call_ids":["senseval2.d002.s052.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":550,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.00584,"input_uncached_usd":0.0027500000000000003,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9748910419875756}
{"item_id":"senseval2.d002.s052.t002","gold_sense_keys":["child%1:18:01::"],"candidates":[{"index":1,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:07::","synset_id":"child.n.04"},{"index":4,"sense_key":"child%1:18:00::","synset_id":"child.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d002.s052.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"child%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":192,"reasoning_output_tokens":176},"cost":{"currency":"USD","total_usd":0.008190000000000001,"input_uncached_usd":0.0024300000000000003,"input_cached_usd":0.0,"output_usd":0.00576,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.90345029195305}
{"item_id":"senseval2.d002.s052.t003","gold_sense_keys":["education%1:09:01::"],"candidates":[{"index":1,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":2,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"},{"index":5,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":6,"sense_key":"education%1:07:01::","synset_id":"education.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s052.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"education%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01873,"input_uncached_usd":0.00289,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.732336499961093}
{"item_id":"senseval2.d002.s052.t004","gold_sense_keys":["home%1:15:04::"],"candidates":[{"index":1,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":2,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"},{"index":3,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":4,"sense_key":"home%1:06:01::","synset_id":"home.n.09"},{"index":5,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":6,"sense_key":"home%1:14:00::","synset_id":"family.n.01"},{"index":7,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"},{"index":8,"sense_key":"home%1:15:02::","synset_id":"home.n.03"},{"index":9,"sense_key":"home%1:15:03::","synset_id":"home.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"home%1:15:04::","call_ids":["senseval2.d002.s052.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"home%1:15:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":422,"reasoning_output_tokens":407},"cost":{"currency":"USD","total_usd":0.016275,"input_uncached_usd":0.003615,"input_cached_usd":0.0,"output_usd":0.012660000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.273243625066243}
{"item_id":"senseval2.d002.s053.t000","gold_sense_keys":["see%2:31:03::"],"candidates":[{"index":1,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":2,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":3,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"},{"index":4,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":5,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":6,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":7,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":8,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":9,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":10,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":11,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":12,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":15,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":16,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":17,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":18,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":19,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":20,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":23,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":24,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"see%2:31:03::","call_ids":["senseval2.d002.s053.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"see%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1450,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.01031,"input_uncached_usd":0.00725,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3194320829352364}
{"item_id":"senseval2.d002.s053.t001","gold_sense_keys":["kid%1:18:01::"],"candidates":[{"index":1,"sense_key":"kid%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"kid%1:27:00::","synset_id":"kid.n.02"},{"index":3,"sense_key":"kid%1:18:02::","synset_id":"kyd.n.01"},{"index":4,"sense_key":"kid%1:05:00::","synset_id":"kid.n.05"},{"index":5,"sense_key":"kid%1:18:01::","synset_id":"child.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"kid%1:18:01::","call_ids":["senseval2.d002.s053.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"kid%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01834,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.489159832941368}
{"item_id":"senseval2.d002.s053.t002","gold_sense_keys":["play%2:41:02::"],"candidates":[{"index":1,"sense_key":"play%2:38:03::","synset_id":"play.v.24"},{"index":2,"sense_key":"play%2:33:01::","synset_id":"play.v.10"},{"index":3,"sense_key":"play%2:36:00::","synset_id":"play.v.06"},{"index":4,"sense_key":"play%2:36:13::","synset_id":"bring.v.03"},{"index":5,"sense_key":"play%2:36:03::","synset_id":"play.v.14"},{"index":6,"sense_key":"play%2:29:04::","synset_id":"play.v.35"},{"index":7,"sense_key":"play%2:29:00::","synset_id":"play.v.16"},{"index":8,"sense_key":"play%2:41:02::","synset_id":"play.v.17"},{"index":9,"sense_key":"play%2:36:05::","synset_id":"play.v.18"},{"index":10,"sense_key":"play%2:41:00::","synset_id":"play.v.11"},{"index":11,"sense_key":"play%2:34:13::","synset_id":"play.v.20"},{"index":12,"sense_key":"play%2:36:10::","synset_id":"act.v.10"},{"index":13,"sense_key":"play%2:35:07::","synset_id":"play.v.28"},{"index":14,"sense_key":"play%2:41:03::","synset_id":"play.v.05"},{"index":15,"sense_key":"play%2:36:02::","synset_id":"act.v.03"},{"index":16,"sense_key":"play%2:36:04::","synset_id":"play.v.26"},{"index":17,"sense_key":"play%2:36:12::","synset_id":"play.v.03"},{"index":18,"sense_key":"play%2:35:00::","synset_id":"toy.v.02"},{"index":19,"sense_key":"play%2:36:06::","synset_id":"play.v.12"},{"index":20,"sense_key":"play%2:42:00::","synset_id":"play.v.22"},{"index":21,"sense_key":"play%2:33:13::","synset_id":"play.v.32"},{"index":22,"sense_key":"play%2:38:00::","synset_id":"play.v.09"},{"index":23,"sense_key":"play%2:33:03::","synset_id":"play.v.15"},{"index":24,"sense_key":"play%2:41:12::","synset_id":"play.v.02"},{"index":25,"sense_key":"play%2:33:09::","synset_id":"play.v.31"},{"index":26,"sense_key":"play%2:33:08::","synset_id":"bet.v.02"},{"index":27,"sense_key":"play%2:33:12::","synset_id":"play.v.33"},{"index":28,"sense_key":"play%2:33:07::","synset_id":"play.v.29"},{"index":29,"sense_key":"play%2:36:11::","synset_id":"play.v.13"},{"index":30,"sense_key":"play%2:41:13::","synset_id":"dally.v.01"},{"index":31,"sense_key":"play%2:33:02::","synset_id":"meet.v.10"},{"index":32,"sense_key":"play%2:29:01::","synset_id":"act.v.05"},{"index":33,"sense_key":"play%2:33:00::","synset_id":"play.v.01"},{"index":34,"sense_key":"play%2:31:01::","synset_id":"dally.v.04"},{"index":35,"sense_key":"play%2:36:01::","synset_id":"play.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"play%2:41:02::","call_ids":["senseval2.d002.s053.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"play%2:41:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1843,"cached_input_tokens":0,"output_tokens":322,"reasoning_output_tokens":306},"cost":{"currency":"USD","total_usd":0.018875000000000003,"input_uncached_usd":0.009215000000000001,"input_cached_usd":0.0,"output_usd":0.00966,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.090540708974004}
{"item_id":"senseval2.d002.s053.t006","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d002.s053.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"child%1:18:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":1531,"reasoning_output_tokens":1503},"cost":{"currency":"USD","total_usd":0.04833,"input_uncached_usd":0.0024000000000000002,"input_cached_usd":0.0,"output_usd":0.04593,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":22.282441708026454}
{"item_id":"senseval2.d002.s053.t007","gold_sense_keys":["spend%2:42:00::"],"candidates":[{"index":1,"sense_key":"spend%2:42:00::","synset_id":"spend.v.01"},{"index":2,"sense_key":"spend%2:40:01::","synset_id":"spend.v.03"},{"index":3,"sense_key":"spend%2:40:00::","synset_id":"spend.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"spend%2:42:00::","call_ids":["senseval2.d002.s053.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"spend%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.00355,"input_uncached_usd":0.00205,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4335120409959927}
{"item_id":"senseval2.d002.s053.t009","gold_sense_keys":["time%1:28:05::"],"candidates":[{"index":1,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"},{"index":2,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":3,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":4,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":5,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":6,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":7,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":8,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":9,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":10,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"time%1:28:05::","call_ids":["senseval2.d002.s053.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"time%1:28:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.006135,"input_uncached_usd":0.0035550000000000004,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6094174159225076}
{"item_id":"senseval2.d002.s053.t011","gold_sense_keys":["scrutinize%2:39:00::"],"candidates":[{"index":1,"sense_key":"scrutinize%2:39:00::","synset_id":"size_up.v.01"},{"index":2,"sense_key":"scrutinize%2:31:00::","synset_id":"audit.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"scrutinize%2:39:00::","call_ids":["senseval2.d002.s053.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"scrutinize%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":122},"cost":{"currency":"USD","total_usd":0.006175000000000001,"input_uncached_usd":0.0020350000000000004,"input_cached_usd":0.0,"output_usd":0.0041400000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.876429499941878}
{"item_id":"senseval2.d002.s054.t000","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"},{"index":2,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s054.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":377,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.003325,"input_uncached_usd":0.0018850000000000002,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1960400419775397}
{"item_id":"senseval2.d002.s054.t002","gold_sense_keys":["option%1:09:00::"],"candidates":[{"index":1,"sense_key":"option%1:09:00::","synset_id":"option.n.02"},{"index":2,"sense_key":"option%1:04:00::","synset_id":"choice.n.02"},{"index":3,"sense_key":"option%1:21:00::","synset_id":"option.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"option%1:09:00::","call_ids":["senseval2.d002.s054.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"option%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":109},"cost":{"currency":"USD","total_usd":0.00594,"input_uncached_usd":0.00222,"input_cached_usd":0.0,"output_usd":0.00372,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5275291659636423}
{"item_id":"senseval2.d002.s054.t003","gold_sense_keys":["switch%2:30:02::"],"candidates":[{"index":1,"sense_key":"switch%2:40:00::","synset_id":"trade.v.04"},{"index":2,"sense_key":"switch%2:30:03::","synset_id":"switch_over.v.01"},{"index":3,"sense_key":"switch%2:35:00::","synset_id":"throw.v.06"},{"index":4,"sense_key":"switch%2:35:03::","synset_id":"switch.v.06"},{"index":5,"sense_key":"switch%2:30:00::","synset_id":"interchange.v.04"},{"index":6,"sense_key":"switch%2:30:02::","synset_id":"switch.v.03"},{"index":7,"sense_key":"switch%2:30:01::","synset_id":"switch.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"switch%2:30:02::","call_ids":["senseval2.d002.s054.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"switch%2:30:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.00547,"input_uncached_usd":0.00289,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.103091917000711}
{"item_id":"senseval2.d002.s055.t000","gold_sense_keys":["community%1:14:00::"],"candidates":[{"index":1,"sense_key":"community%1:14:04::","synset_id":"community.n.03"},{"index":2,"sense_key":"community%1:15:00::","synset_id":"residential_district.n.01"},{"index":3,"sense_key":"community%1:26:00::","synset_id":"community.n.04"},{"index":4,"sense_key":"community%1:14:00::","synset_id":"community.n.01"},{"index":5,"sense_key":"community%1:21:00::","synset_id":"community.n.02"},{"index":6,"sense_key":"community%1:14:02::","synset_id":"community.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"community%1:14:00::","call_ids":["senseval2.d002.s055.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"community%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":507,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.006015,"input_uncached_usd":0.0025350000000000004,"input_cached_usd":0.0,"output_usd":0.00348,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1450241659767926}
{"item_id":"senseval2.d002.s055.t002","gold_sense_keys":["even%4:02:01::"],"candidates":[{"index":1,"sense_key":"even%4:02:00::","synset_id":"even.r.01"},{"index":2,"sense_key":"even%4:02:04::","synset_id":"even.r.04"},{"index":3,"sense_key":"even%4:02:01::","synset_id":"even.r.03"},{"index":4,"sense_key":"even%4:02:03::","synset_id":"even.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"even%4:02:01::","call_ids":["senseval2.d002.s055.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"even%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004035,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.034545250004157}
{"item_id":"senseval2.d002.s055.t003","gold_sense_keys":["worse%3:00:00::"],"candidates":[{"index":1,"sense_key":"bad%5:00:00:nonstandard:03","synset_id":"bad.s.06"},{"index":2,"sense_key":"bad%3:00:00::","synset_id":"bad.a.01"},{"index":3,"sense_key":"bad%5:00:02:unsound:00","synset_id":"bad.s.07"},{"index":4,"sense_key":"bad%5:00:00:malfunctioning:00","synset_id":"bad.s.12"},{"index":5,"sense_key":"bad%5:00:00:evil:00","synset_id":"bad.s.10"},{"index":6,"sense_key":"bad%5:00:00:uncomfortable:00","synset_id":"bad.s.02"},{"index":7,"sense_key":"bad%5:00:00:intense:00","synset_id":"bad.s.01"},{"index":8,"sense_key":"bad%5:00:00:counterfeit:00","synset_id":"bad.s.11"},{"index":9,"sense_key":"bad%5:00:00:invalid:00","synset_id":"bad.s.04"},{"index":10,"sense_key":"bad%5:00:00:unhealthy:00","synset_id":"bad.s.08"},{"index":11,"sense_key":"regretful%3:00:00::","synset_id":"regretful.a.01"},{"index":12,"sense_key":"bad%5:00:00:inferior:02","synset_id":"bad.s.05"},{"index":13,"sense_key":"bad%5:00:00:harmful:00","synset_id":"bad.s.09"},{"index":14,"sense_key":"worse%3:00:00::","synset_id":"worse.a.01"},{"index":15,"sense_key":"bad%5:00:00:stale:00","synset_id":"bad.s.03"},{"index":16,"sense_key":"worse%3:00:02::","synset_id":"worse.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"worse%3:00:00::","call_ids":["senseval2.d002.s055.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"worse%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1022,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.007000000000000001,"input_uncached_usd":0.005110000000000001,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.223040792043321}
{"item_id":"senseval2.d002.s055.t004","gold_sense_keys":["idea%1:09:00::"],"candidates":[{"index":1,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"},{"index":2,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"},{"index":3,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"},{"index":4,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"},{"index":5,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"idea%1:09:00::","call_ids":["senseval2.d002.s055.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"idea%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":101},"cost":{"currency":"USD","total_usd":0.00613,"input_uncached_usd":0.00265,"input_cached_usd":0.0,"output_usd":0.00348,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3992468339856714}
{"item_id":"senseval2.d002.s056.t000","gold_sense_keys":["here%4:02:02::"],"candidates":[{"index":1,"sense_key":"here%4:02:00::","synset_id":"here.r.01"},{"index":2,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":3,"sense_key":"here%4:02:04::","synset_id":"here.r.04"},{"index":4,"sense_key":"here%4:02:02::","synset_id":"here.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"here%4:02:02::","call_ids":["senseval2.d002.s056.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"here%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.005015,"input_uncached_usd":0.0021950000000000003,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9251326250378042}
{"item_id":"senseval2.d002.s057.t000","gold_sense_keys":["locally%4:02:00::"],"candidates":[{"index":1,"sense_key":"locally%4:02:01::","synset_id":"locally.r.02"},{"index":2,"sense_key":"locally%4:02:00::","synset_id":"locally.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"locally%4:02:00::","call_ids":["senseval2.d002.s057.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"locally%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":377,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.0035050000000000003,"input_uncached_usd":0.0018850000000000002,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.594575709081255}
{"item_id":"senseval2.d002.s057.t003","gold_sense_keys":["especially%4:02:00::"],"candidates":[{"index":1,"sense_key":"especially%4:02:01::","synset_id":"specially.r.01"},{"index":2,"sense_key":"especially%4:02:00::","synset_id":"particularly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"especially%4:02:00::","call_ids":["senseval2.d002.s057.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"especially%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":392,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.003940000000000001,"input_uncached_usd":0.0019600000000000004,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5335594590287656}
{"item_id":"senseval2.d002.s057.t005","gold_sense_keys":["city%1:15:00::"],"candidates":[{"index":1,"sense_key":"city%1:15:00::","synset_id":"city.n.01"},{"index":2,"sense_key":"city%1:14:00::","synset_id":"city.n.03"},{"index":3,"sense_key":"city%1:15:01::","synset_id":"city.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"city%1:15:00::","call_ids":["senseval2.d002.s057.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"city%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":386},"cost":{"currency":"USD","total_usd":0.014190000000000001,"input_uncached_usd":0.00216,"input_cached_usd":0.0,"output_usd":0.01203,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.533121667103842}
{"item_id":"senseval2.d002.s057.t006","gold_sense_keys":["become%2:42:01::"],"candidates":[{"index":1,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":2,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"become%2:42:03::","synset_id":"become.v.04"},{"index":4,"sense_key":"become%2:42:00::","synset_id":"become.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"become%2:42:01::","call_ids":["senseval2.d002.s057.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"become%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":452,"reasoning_output_tokens":437},"cost":{"currency":"USD","total_usd":0.01593,"input_uncached_usd":0.00237,"input_cached_usd":0.0,"output_usd":0.013560000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.244568875059485}
{"item_id":"senseval2.d002.s057.t007","gold_sense_keys":["ambitious%3:00:00::"],"candidates":[{"index":1,"sense_key":"ambitious%3:00:00::","synset_id":"ambitious.a.01"},{"index":2,"sense_key":"ambitious%5:00:00:difficult:00","synset_id":"ambitious.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ambitious%3:00:00::","call_ids":["senseval2.d002.s057.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ambitious%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":385,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.003845,"input_uncached_usd":0.001925,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.755747582996264}
{"item_id":"senseval2.d002.s057.t008","gold_sense_keys":["generally%4:02:02::"],"candidates":[{"index":1,"sense_key":"generally%4:02:02::","synset_id":"by_and_large.r.01"},{"index":2,"sense_key":"generally%4:02:00::","synset_id":"broadly.r.01"},{"index":3,"sense_key":"generally%4:02:01::","synset_id":"generally.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"generally%4:02:02::","call_ids":["senseval2.d002.s057.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"generally%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":103},"cost":{"currency":"USD","total_usd":0.005755,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.00354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2600752090802416}
{"item_id":"senseval2.d002.s057.t009","gold_sense_keys":["corrupt%3:00:00::","corrupt%3:00:04::"],"candidates":[{"index":1,"sense_key":"corrupt%5:00:00:stale:00","synset_id":"corrupt.s.02"},{"index":2,"sense_key":"corrupt%3:00:04::","synset_id":"crooked.a.02"},{"index":3,"sense_key":"corrupt%5:00:00:imperfect:00","synset_id":"corrupt.s.01"},{"index":4,"sense_key":"corrupt%3:00:00::","synset_id":"corrupt.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"corrupt%3:00:00::","call_ids":["senseval2.d002.s057.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"corrupt%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01823,"input_uncached_usd":0.00239,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.821451249998063}
{"item_id":"senseval2.d002.s057.t012","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":2,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:00:01::","call_ids":["senseval2.d002.s057.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":133},"cost":{"currency":"USD","total_usd":0.006610000000000001,"input_uncached_usd":0.00214,"input_cached_usd":0.0,"output_usd":0.00447,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.993511833017692}
{"item_id":"senseval2.d002.s057.t013","gold_sense_keys":["politician%1:18:00::","politician%1:18:01::"],"candidates":[{"index":1,"sense_key":"politician%1:18:00::","synset_id":"politician.n.02"},{"index":2,"sense_key":"politician%1:18:02::","synset_id":"politician.n.03"},{"index":3,"sense_key":"politician%1:18:01::","synset_id":"politician.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"politician%1:18:00::","call_ids":["senseval2.d002.s057.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"politician%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":401,"cached_input_tokens":0,"output_tokens":973,"reasoning_output_tokens":951},"cost":{"currency":"USD","total_usd":0.031195,"input_uncached_usd":0.0020050000000000003,"input_cached_usd":0.0,"output_usd":0.02919,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.297140333917923}
{"item_id":"senseval2.d002.s057.t015","gold_sense_keys":["politician%1:18:00::","politician%1:18:01::"],"candidates":[{"index":1,"sense_key":"politician%1:18:00::","synset_id":"politician.n.02"},{"index":2,"sense_key":"politician%1:18:01::","synset_id":"politician.n.01"},{"index":3,"sense_key":"politician%1:18:02::","synset_id":"politician.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"politician%1:18:00::","call_ids":["senseval2.d002.s057.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"politician%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":271,"reasoning_output_tokens":255},"cost":{"currency":"USD","total_usd":0.01013,"input_uncached_usd":0.002,"input_cached_usd":0.0,"output_usd":0.00813,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.3555842080386356}
{"item_id":"senseval2.d002.s058.t000","gold_sense_keys":["process%1:04:00::"],"candidates":[{"index":1,"sense_key":"process%1:10:00::","synset_id":"summons.n.03"},{"index":2,"sense_key":"process%1:04:00::","synset_id":"procedure.n.01"},{"index":3,"sense_key":"process%1:09:00::","synset_id":"process.n.02"},{"index":4,"sense_key":"process%1:03:00::","synset_id":"process.n.06"},{"index":5,"sense_key":"process%1:08:00::","synset_id":"process.n.05"},{"index":6,"sense_key":"process%1:09:01::","synset_id":"process.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"process%1:03:00::","call_ids":["senseval2.d002.s058.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"process%1:03:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01894,"input_uncached_usd":0.0031000000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.286679084063508}
{"item_id":"senseval2.d002.s058.t001","gold_sense_keys":["try%2:41:00::"],"candidates":[{"index":1,"sense_key":"try%2:41:05::","synset_id":"hear.v.03"},{"index":2,"sense_key":"try%2:29:00::","synset_id":"try_on.v.01"},{"index":3,"sense_key":"try%2:30:00::","synset_id":"try.v.08"},{"index":4,"sense_key":"try%2:41:01::","synset_id":"test.v.01"},{"index":5,"sense_key":"try%2:37:00::","synset_id":"try.v.07"},{"index":6,"sense_key":"try%2:41:06::","synset_id":"judge.v.05"},{"index":7,"sense_key":"try%2:34:00::","synset_id":"sample.v.01"},{"index":8,"sense_key":"try%2:37:01::","synset_id":"try.v.06"},{"index":9,"sense_key":"try%2:41:00::","synset_id":"try.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"try%2:41:00::","call_ids":["senseval2.d002.s058.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"try%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.005865,"input_uncached_usd":0.003765,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.108109083026648}
{"item_id":"senseval2.d002.s058.t002","gold_sense_keys":["system%1:09:00::"],"candidates":[{"index":1,"sense_key":"system%1:08:00::","synset_id":"system.n.06"},{"index":2,"sense_key":"system%1:14:00::","synset_id":"system.n.02"},{"index":3,"sense_key":"system%1:27:00::","synset_id":"system.n.03"},{"index":4,"sense_key":"system%1:06:00::","synset_id":"system.n.01"},{"index":5,"sense_key":"system%1:09:02::","synset_id":"arrangement.n.03"},{"index":6,"sense_key":"system%1:08:01::","synset_id":"system.n.08"},{"index":7,"sense_key":"system%1:09:00::","synset_id":"system.n.04"},{"index":8,"sense_key":"system%1:09:01::","synset_id":"system.n.07"},{"index":9,"sense_key":"system%1:07:00::","synset_id":"organization.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"system%1:09:00::","call_ids":["senseval2.d002.s058.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"system%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":484,"reasoning_output_tokens":468},"cost":{"currency":"USD","total_usd":0.018375,"input_uncached_usd":0.0038550000000000004,"input_cached_usd":0.0,"output_usd":0.01452,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.601440791040659}
{"item_id":"senseval2.d002.s058.t003","gold_sense_keys":["school%1:14:00::"],"candidates":[{"index":1,"sense_key":"school%1:14:00::","synset_id":"school.n.01"},{"index":2,"sense_key":"school%1:28:00::","synset_id":"school.n.05"},{"index":3,"sense_key":"school%1:06:00::","synset_id":"school.n.02"},{"index":4,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":5,"sense_key":"school%1:14:01::","synset_id":"school.n.07"},{"index":6,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":7,"sense_key":"school%1:09:00::","synset_id":"school.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"school%1:14:00::","call_ids":["senseval2.d002.s058.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"school%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":301,"reasoning_output_tokens":285},"cost":{"currency":"USD","total_usd":0.0121,"input_uncached_usd":0.0030700000000000002,"input_cached_usd":0.0,"output_usd":0.00903,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.359546541934833}
{"item_id":"senseval2.d002.s058.t004","gold_sense_keys":["governance%1:04:00::"],"candidates":[{"index":1,"sense_key":"governance%1:14:00::","synset_id":"administration.n.02"},{"index":2,"sense_key":"governance%1:04:00::","synset_id":"government.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"governance%1:04:00::","call_ids":["senseval2.d002.s058.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"governance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":381,"reasoning_output_tokens":365},"cost":{"currency":"USD","total_usd":0.013685,"input_uncached_usd":0.002255,"input_cached_usd":0.0,"output_usd":0.011430000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.900194457964972}
{"item_id":"senseval2.d002.s058.t006","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":4,"sense_key":"other%3:00:00::","synset_id":"other.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d002.s058.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.004530000000000001,"input_uncached_usd":0.0024600000000000004,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.329899250064045}
{"item_id":"senseval2.d002.s058.t007","gold_sense_keys":["city%1:15:01::"],"candidates":[{"index":1,"sense_key":"city%1:15:01::","synset_id":"city.n.02"},{"index":2,"sense_key":"city%1:14:00::","synset_id":"city.n.03"},{"index":3,"sense_key":"city%1:15:00::","synset_id":"city.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"city%1:15:01::","call_ids":["senseval2.d002.s058.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"city%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":321,"reasoning_output_tokens":306},"cost":{"currency":"USD","total_usd":0.011840000000000002,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.00963,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.408302958006971}
{"item_id":"senseval2.d002.s058.t008","gold_sense_keys":["institute%2:36:01::"],"candidates":[{"index":1,"sense_key":"institute%2:36:00::","synset_id":"institute.v.02"},{"index":2,"sense_key":"institute%2:36:01::","synset_id":"establish.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"institute%2:36:01::","call_ids":["senseval2.d002.s058.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"institute%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":398,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.00415,"input_uncached_usd":0.00199,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8440548330545425}
{"item_id":"senseval2.d002.s059.t000","gold_sense_keys":["most%3:00:02::"],"candidates":[{"index":1,"sense_key":"most%3:00:02::","synset_id":"most.a.01"},{"index":2,"sense_key":"most%3:00:01::","synset_id":"most.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"most%3:00:02::","call_ids":["senseval2.d002.s059.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"most%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":98},"cost":{"currency":"USD","total_usd":0.005520000000000001,"input_uncached_usd":0.0021300000000000004,"input_cached_usd":0.0,"output_usd":0.0033900000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3265474169747904}
{"item_id":"senseval2.d002.s059.t002","gold_sense_keys":["expenditure%1:21:00::"],"candidates":[{"index":1,"sense_key":"expenditure%1:04:01::","synset_id":"consumption.n.04"},{"index":2,"sense_key":"expenditure%1:04:00::","synset_id":"expending.n.01"},{"index":3,"sense_key":"expenditure%1:21:00::","synset_id":"outgo.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"expenditure%1:21:00::","call_ids":["senseval2.d002.s059.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"expenditure%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.004775,"input_uncached_usd":0.0020150000000000003,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6404904590453953}
{"item_id":"senseval2.d002.s059.t003","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":2,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":3,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":4,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s059.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":107},"cost":{"currency":"USD","total_usd":0.006535000000000001,"input_uncached_usd":0.0028450000000000003,"input_cached_usd":0.0,"output_usd":0.00369,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.261280208011158}
{"item_id":"senseval2.d002.s059.t005","gold_sense_keys":["circumstance%1:26:01::","circumstance%1:26:02::"],"candidates":[{"index":1,"sense_key":"circumstance%1:11:00::","synset_id":"circumstance.n.04"},{"index":2,"sense_key":"circumstance%1:09:00::","synset_id":"circumstance.n.03"},{"index":3,"sense_key":"circumstance%1:26:01::","synset_id":"circumstance.n.01"},{"index":4,"sense_key":"circumstance%1:26:02::","synset_id":"context.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"circumstance%1:26:01::","call_ids":["senseval2.d002.s059.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"circumstance%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":3801,"reasoning_output_tokens":3746},"cost":{"currency":"USD","total_usd":0.116285,"input_uncached_usd":0.002255,"input_cached_usd":0.0,"output_usd":0.11403,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":60.26949999993667}
{"item_id":"senseval2.d002.s059.t006","gold_sense_keys":["probably%4:02:00::"],"candidates":[{"index":1,"sense_key":"probably%4:02:00::","synset_id":"probably.r.01"},{"index":2,"sense_key":"probably%4:02:01::","synset_id":"credibly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"probably%4:02:00::","call_ids":["senseval2.d002.s059.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"probably%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":159,"reasoning_output_tokens":144},"cost":{"currency":"USD","total_usd":0.006845,"input_uncached_usd":0.002075,"input_cached_usd":0.0,"output_usd":0.00477,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.016209249966778}
{"item_id":"senseval2.d002.s059.t007","gold_sense_keys":["make%2:30:00::"],"candidates":[{"index":1,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":2,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":3,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":4,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":5,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":6,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":7,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":8,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":9,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":10,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":11,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":12,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":15,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":16,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":17,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":18,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":19,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":20,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":21,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":22,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":23,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":24,"sense_key":"make%2:29:01::","synset_id":"make.v.49"},{"index":25,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":26,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":27,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":28,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":29,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":30,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":31,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":32,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":33,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":34,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":35,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":36,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":37,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":38,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":39,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":40,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":41,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":42,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":43,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":44,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":45,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":46,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":47,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":48,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":49,"sense_key":"make%2:40:02::","synset_id":"have.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"make%2:30:00::","call_ids":["senseval2.d002.s059.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"make%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2260,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02714,"input_uncached_usd":0.011300000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.437613791087642}
{"item_id":"senseval2.d002.s059.t008","gold_sense_keys":["thing%1:09:02::"],"candidates":[{"index":1,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":2,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":3,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":4,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":5,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":6,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":7,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":8,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"},{"index":9,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":10,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":11,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":12,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"thing%1:26:00::","call_ids":["senseval2.d002.s059.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"thing%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":700,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01934,"input_uncached_usd":0.0035,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.207829708000645}
{"item_id":"senseval2.d002.s059.t009","gold_sense_keys":["worse%3:00:00::"],"candidates":[{"index":1,"sense_key":"bad%5:00:02:unsound:00","synset_id":"bad.s.07"},{"index":2,"sense_key":"bad%5:00:00:unhealthy:00","synset_id":"bad.s.08"},{"index":3,"sense_key":"bad%5:00:00:intense:00","synset_id":"bad.s.01"},{"index":4,"sense_key":"bad%3:00:00::","synset_id":"bad.a.01"},{"index":5,"sense_key":"worse%3:00:00::","synset_id":"worse.a.01"},{"index":6,"sense_key":"bad%5:00:00:stale:00","synset_id":"bad.s.03"},{"index":7,"sense_key":"bad%5:00:00:nonstandard:03","synset_id":"bad.s.06"},{"index":8,"sense_key":"bad%5:00:00:evil:00","synset_id":"bad.s.10"},{"index":9,"sense_key":"bad%5:00:00:inferior:02","synset_id":"bad.s.05"},{"index":10,"sense_key":"bad%5:00:00:uncomfortable:00","synset_id":"bad.s.02"},{"index":11,"sense_key":"bad%5:00:00:harmful:00","synset_id":"bad.s.09"},{"index":12,"sense_key":"worse%3:00:02::","synset_id":"worse.a.02"},{"index":13,"sense_key":"bad%5:00:00:invalid:00","synset_id":"bad.s.04"},{"index":14,"sense_key":"regretful%3:00:00::","synset_id":"regretful.a.01"},{"index":15,"sense_key":"bad%5:00:00:malfunctioning:00","synset_id":"bad.s.12"},{"index":16,"sense_key":"bad%5:00:00:counterfeit:00","synset_id":"bad.s.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"worse%3:00:00::","call_ids":["senseval2.d002.s059.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"worse%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1037,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.0073750000000000005,"input_uncached_usd":0.005185,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7882758330088109}
{"item_id":"senseval2.d002.s059.t010","gold_sense_keys":["better%3:00:00::"],"candidates":[{"index":1,"sense_key":"good%5:00:00:genuine:00","synset_id":"good.s.17"},{"index":2,"sense_key":"good%5:00:00:healthful:00","synset_id":"good.s.16"},{"index":3,"sense_key":"good%5:00:00:fresh:01","synset_id":"good.s.18"},{"index":4,"sense_key":"well%5:00:00:advisable:00","synset_id":"well.s.02"},{"index":5,"sense_key":"better%3:00:02::","synset_id":"better.a.02"},{"index":6,"sense_key":"good%5:00:00:complete:00","synset_id":"good.s.07"},{"index":7,"sense_key":"good%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":8,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":9,"sense_key":"better%5:00:00:advisable:00","synset_id":"better.s.01"},{"index":10,"sense_key":"good%5:00:00:pleasing:00","synset_id":"good.s.13"},{"index":11,"sense_key":"good%5:00:00:opportune:00","synset_id":"good.s.10"},{"index":12,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":13,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":14,"sense_key":"good%3:00:01::","synset_id":"good.a.01"},{"index":15,"sense_key":"good%5:00:00:righteous:00","synset_id":"good.s.05"},{"index":16,"sense_key":"better%5:00:00:major:01","synset_id":"better.s.02"},{"index":17,"sense_key":"good%5:00:00:discriminating:00","synset_id":"good.s.19"},{"index":18,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":19,"sense_key":"estimable%5:00:00:reputable:00","synset_id":"estimable.s.01"},{"index":20,"sense_key":"dear%5:00:00:close:02","synset_id":"dear.s.02"},{"index":21,"sense_key":"beneficial%5:00:00:advantageous:00","synset_id":"beneficial.s.01"},{"index":22,"sense_key":"better%3:00:00::","synset_id":"better.a.01"},{"index":23,"sense_key":"dependable%5:00:00:sound:00","synset_id":"dependable.s.03"},{"index":24,"sense_key":"good%5:00:00:healthy:00","synset_id":"good.s.15"},{"index":25,"sense_key":"good%3:00:02::","synset_id":"good.a.03"},{"index":26,"sense_key":"well%3:00:01::","synset_id":"well.a.01"},{"index":27,"sense_key":"good%5:00:00:nice:00","synset_id":"good.s.04"},{"index":28,"sense_key":"adept%5:00:00:skilled:00","synset_id":"adept.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":22,"chosen_sense_key":"better%3:00:00::","call_ids":["senseval2.d002.s059.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":22,"predicted_sense_key":"better%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1580,"cached_input_tokens":0,"output_tokens":228,"reasoning_output_tokens":212},"cost":{"currency":"USD","total_usd":0.014740000000000001,"input_uncached_usd":0.0079,"input_cached_usd":0.0,"output_usd":0.006840000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.611429082928225}
{"item_id":"senseval2.d002.s060.t000","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::","reason%1:16:00::"],"candidates":[{"index":1,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":2,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":3,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":4,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":5,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":6,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reason%1:10:01::","call_ids":["senseval2.d002.s060.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reason%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":2081,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.065435,"input_uncached_usd":0.0030050000000000003,"input_cached_usd":0.0,"output_usd":0.06243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":29.416972832987085}
{"item_id":"senseval2.d002.s060.t001","gold_sense_keys":["simple%3:00:02::","simple%5:00:00:easy:01"],"candidates":[{"index":1,"sense_key":"simple%3:00:02::","synset_id":"simple.a.01"},{"index":2,"sense_key":"simple%5:00:00:retarded:00","synset_id":"dim-witted.s.01"},{"index":3,"sense_key":"simple%5:00:00:naive:00","synset_id":"childlike.s.02"},{"index":4,"sense_key":"simple%3:00:01::","synset_id":"simple.a.06"},{"index":5,"sense_key":"simple%5:00:00:easy:01","synset_id":"elementary.s.01"},{"index":6,"sense_key":"simple%5:00:02:plain:01","synset_id":"bare.s.04"},{"index":7,"sense_key":"simple%5:00:00:plain:01","synset_id":"simple.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"simple%3:00:02::","call_ids":["senseval2.d002.s060.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"simple%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":4153,"reasoning_output_tokens":4094},"cost":{"currency":"USD","total_usd":0.12771,"input_uncached_usd":0.0031200000000000004,"input_cached_usd":0.0,"output_usd":0.12459,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":70.33786891703494}
{"item_id":"senseval2.d002.s060.t002","gold_sense_keys":["education%1:04:00::","education%1:09:01::"],"candidates":[{"index":1,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"},{"index":2,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":3,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":4,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:09:01::","synset_id":"education.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s060.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":246,"reasoning_output_tokens":231},"cost":{"currency":"USD","total_usd":0.010315000000000001,"input_uncached_usd":0.002935,"input_cached_usd":0.0,"output_usd":0.00738,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.478343709022738}
{"item_id":"senseval2.d002.s060.t005","gold_sense_keys":["influence%1:19:00::"],"candidates":[{"index":1,"sense_key":"influence%1:09:00::","synset_id":"influence.n.03"},{"index":2,"sense_key":"influence%1:04:00::","synset_id":"influence.n.02"},{"index":3,"sense_key":"influence%1:18:00::","synset_id":"influence.n.05"},{"index":4,"sense_key":"influence%1:19:00::","synset_id":"influence.n.04"},{"index":5,"sense_key":"influence%1:07:00::","synset_id":"influence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"influence%1:19:00::","call_ids":["senseval2.d002.s060.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"influence%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":136},"cost":{"currency":"USD","total_usd":0.007105000000000001,"input_uncached_usd":0.0025450000000000004,"input_cached_usd":0.0,"output_usd":0.00456,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0084545420249924}
{"item_id":"senseval2.d002.s060.t006","gold_sense_keys":["money%1:21:00::"],"candidates":[{"index":1,"sense_key":"money%1:21:02::","synset_id":"money.n.02"},{"index":2,"sense_key":"money%1:21:00::","synset_id":"money.n.01"},{"index":3,"sense_key":"money%1:21:01::","synset_id":"money.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"money%1:21:02::","call_ids":["senseval2.d002.s060.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"money%1:21:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":430,"cached_input_tokens":0,"output_tokens":239,"reasoning_output_tokens":223},"cost":{"currency":"USD","total_usd":0.00932,"input_uncached_usd":0.00215,"input_cached_usd":0.0,"output_usd":0.00717,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7804580410011113}
{"item_id":"senseval2.d002.s061.t000","gold_sense_keys":["decade%1:28:00::"],"candidates":[{"index":1,"sense_key":"decade%1:28:00::","synset_id":"decade.n.01"},{"index":2,"sense_key":"decade%1:23:00::","synset_id":"ten.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decade%1:28:00::","call_ids":["senseval2.d002.s061.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decade%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.00406,"input_uncached_usd":0.0020800000000000003,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6081914579262957}
{"item_id":"senseval2.d002.s061.t001","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s061.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":399,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.004605,"input_uncached_usd":0.0019950000000000002,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1022984590381384}
{"item_id":"senseval2.d002.s061.t003","gold_sense_keys":["tell%2:32:00::"],"candidates":[{"index":1,"sense_key":"tell%2:32:02::","synset_id":"order.v.01"},{"index":2,"sense_key":"tell%2:31:00::","synset_id":"tell.v.05"},{"index":3,"sense_key":"tell%2:31:01::","synset_id":"distinguish.v.01"},{"index":4,"sense_key":"tell%2:32:03::","synset_id":"tell.v.07"},{"index":5,"sense_key":"tell%2:32:04::","synset_id":"state.v.01"},{"index":6,"sense_key":"tell%2:32:01::","synset_id":"tell.v.03"},{"index":7,"sense_key":"tell%2:32:00::","synset_id":"tell.v.02"},{"index":8,"sense_key":"tell%2:32:08::","synset_id":"assure.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"tell%2:32:00::","call_ids":["senseval2.d002.s061.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"tell%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":1564,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.050265000000000004,"input_uncached_usd":0.0033450000000000003,"input_cached_usd":0.0,"output_usd":0.04692,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":32.632248874986544}
{"item_id":"senseval2.d002.s061.t005","gold_sense_keys":["even%4:02:01::"],"candidates":[{"index":1,"sense_key":"even%4:02:00::","synset_id":"even.r.01"},{"index":2,"sense_key":"even%4:02:01::","synset_id":"even.r.03"},{"index":3,"sense_key":"even%4:02:04::","synset_id":"even.r.04"},{"index":4,"sense_key":"even%4:02:03::","synset_id":"even.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"even%4:02:01::","call_ids":["senseval2.d002.s061.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"even%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":5708,"reasoning_output_tokens":5632},"cost":{"currency":"USD","total_usd":0.173745,"input_uncached_usd":0.0025050000000000003,"input_cached_usd":0.0,"output_usd":0.17124,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":115.94449645897839}
{"item_id":"senseval2.d002.s061.t007","gold_sense_keys":["class%1:14:03::"],"candidates":[{"index":1,"sense_key":"class%1:14:05::","synset_id":"class.n.05"},{"index":2,"sense_key":"class%1:14:02::","synset_id":"class.n.07"},{"index":3,"sense_key":"class%1:14:01::","synset_id":"class.n.03"},{"index":4,"sense_key":"class%1:14:03::","synset_id":"class.n.02"},{"index":5,"sense_key":"class%1:07:00::","synset_id":"class.n.08"},{"index":6,"sense_key":"class%1:14:00::","synset_id":"class.n.01"},{"index":7,"sense_key":"class%1:14:04::","synset_id":"class.n.06"},{"index":8,"sense_key":"class%1:04:00::","synset_id":"course.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"class%1:14:03::","call_ids":["senseval2.d002.s061.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"class%1:14:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":674,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.0056500000000000005,"input_uncached_usd":0.00337,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1809249999932945}
{"item_id":"senseval2.d002.s061.t008","gold_sense_keys":["effect%1:19:00::"],"candidates":[{"index":1,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"},{"index":2,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"},{"index":3,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"},{"index":4,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":5,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":6,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"effect%1:19:00::","call_ids":["senseval2.d002.s061.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"effect%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004960000000000001,"input_uncached_usd":0.0029800000000000004,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5886419160524383}
{"item_id":"senseval2.d002.s061.t009","gold_sense_keys":["academic%3:01:00::"],"candidates":[{"index":1,"sense_key":"academic%5:00:00:theoretical:01","synset_id":"academic.s.01"},{"index":2,"sense_key":"academic%3:01:00::","synset_id":"academic.a.01"},{"index":3,"sense_key":"academic%5:00:00:scholarly:00","synset_id":"academic.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"academic%3:01:00::","call_ids":["senseval2.d002.s061.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"academic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.005075,"input_uncached_usd":0.002285,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8899612079840153}
{"item_id":"senseval2.d002.s061.t010","gold_sense_keys":["performance%1:04:00::","performance%1:04:03::"],"candidates":[{"index":1,"sense_key":"performance%1:04:03::","synset_id":"performance.n.04"},{"index":2,"sense_key":"performance%1:22:00::","synset_id":"operation.n.08"},{"index":3,"sense_key":"performance%1:04:01::","synset_id":"performance.n.02"},{"index":4,"sense_key":"performance%1:04:00::","synset_id":"performance.n.03"},{"index":5,"sense_key":"performance%1:10:00::","synset_id":"performance.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"performance%1:04:00::","call_ids":["senseval2.d002.s061.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"performance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":333,"reasoning_output_tokens":318},"cost":{"currency":"USD","total_usd":0.012920000000000001,"input_uncached_usd":0.0029300000000000003,"input_cached_usd":0.0,"output_usd":0.00999,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.797079666983336}
{"item_id":"senseval2.d002.s061.t011","gold_sense_keys":["pupil%1:18:00::","pupil%1:18:01::"],"candidates":[{"index":1,"sense_key":"pupil%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"pupil%1:08:00::","synset_id":"pupil.n.02"},{"index":3,"sense_key":"pupil%1:18:01::","synset_id":"schoolchild.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"pupil%1:18:01::","call_ids":["senseval2.d002.s061.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"pupil%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":218,"reasoning_output_tokens":202},"cost":{"currency":"USD","total_usd":0.00875,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.00654,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.123842749977484}
{"item_id":"senseval2.d002.s061.t013","gold_sense_keys":["desirable%3:00:00::"],"candidates":[{"index":1,"sense_key":"desirable%5:00:00:eligible:00","synset_id":"desirable.s.01"},{"index":2,"sense_key":"desirable%3:00:00::","synset_id":"desirable.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"desirable%3:00:00::","call_ids":["senseval2.d002.s061.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"desirable%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.004280000000000001,"input_uncached_usd":0.0020900000000000003,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9642354589886963}
{"item_id":"senseval2.d002.s061.t014","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":2,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":3,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":4,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d002.s061.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.004495000000000001,"input_uncached_usd":0.0025450000000000004,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.825480749947019}
{"item_id":"senseval2.d002.s061.t015","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::","reason%1:16:00::"],"candidates":[{"index":1,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":2,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"},{"index":3,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":4,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":5,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":6,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"reason%1:10:00::","call_ids":["senseval2.d002.s061.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"reason%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":4408,"reasoning_output_tokens":4344},"cost":{"currency":"USD","total_usd":0.13532,"input_uncached_usd":0.0030800000000000003,"input_cached_usd":0.0,"output_usd":0.13224,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":72.4255698339548}
{"item_id":"senseval2.d002.s062.t000","gold_sense_keys":["new%3:00:00::","new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":2,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":3,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":4,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":5,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":6,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":7,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":8,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":9,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"},{"index":10,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":11,"sense_key":"new%3:00:00::","synset_id":"new.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d002.s062.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":856,"cached_input_tokens":0,"output_tokens":292,"reasoning_output_tokens":276},"cost":{"currency":"USD","total_usd":0.01304,"input_uncached_usd":0.00428,"input_cached_usd":0.0,"output_usd":0.00876,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.022474333061837}
{"item_id":"senseval2.d002.s062.t001","gold_sense_keys":["money%1:21:00::","money%1:21:02::"],"candidates":[{"index":1,"sense_key":"money%1:21:00::","synset_id":"money.n.01"},{"index":2,"sense_key":"money%1:21:01::","synset_id":"money.n.03"},{"index":3,"sense_key":"money%1:21:02::","synset_id":"money.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"money%1:21:00::","call_ids":["senseval2.d002.s062.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"money%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01825,"input_uncached_usd":0.0024100000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":17.50075004203245}
{"item_id":"senseval2.d002.s062.t002","gold_sense_keys":["flow%2:38:01::"],"candidates":[{"index":1,"sense_key":"flow%2:35:04::","synset_id":"hang.v.05"},{"index":2,"sense_key":"flow%2:38:01::","synset_id":"flow.v.01"},{"index":3,"sense_key":"flow%2:38:02::","synset_id":"flow.v.03"},{"index":4,"sense_key":"flow%2:38:00::","synset_id":"run.v.06"},{"index":5,"sense_key":"flow%2:29:00::","synset_id":"menstruate.v.01"},{"index":6,"sense_key":"flow%2:30:00::","synset_id":"flow.v.06"},{"index":7,"sense_key":"flow%2:42:00::","synset_id":"flow.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"flow%2:38:01::","call_ids":["senseval2.d002.s062.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"flow%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":170,"reasoning_output_tokens":154},"cost":{"currency":"USD","total_usd":0.008230000000000001,"input_uncached_usd":0.0031300000000000004,"input_cached_usd":0.0,"output_usd":0.0051,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.352125957957469}
{"item_id":"senseval2.d002.s062.t007","gold_sense_keys":["pile%2:35:03::"],"candidates":[{"index":1,"sense_key":"pile%2:38:00::","synset_id":"throng.v.01"},{"index":2,"sense_key":"pile%2:35:00::","synset_id":"stack.v.02"},{"index":3,"sense_key":"pile%2:35:03::","synset_id":"pile.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"pile%2:35:03::","call_ids":["senseval2.d002.s062.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"pile%2:35:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.004705000000000001,"input_uncached_usd":0.0024850000000000002,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.327407540986314}
{"item_id":"senseval2.d002.s062.t008","gold_sense_keys":["more%3:00:01::"],"candidates":[{"index":1,"sense_key":"more%3:00:02::","synset_id":"more.a.02"},{"index":2,"sense_key":"more%3:00:01::","synset_id":"more.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"more%3:00:01::","call_ids":["senseval2.d002.s062.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"more%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":96},"cost":{"currency":"USD","total_usd":0.005725000000000001,"input_uncached_usd":0.0023650000000000003,"input_cached_usd":0.0,"output_usd":0.00336,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.139187957975082}
{"item_id":"senseval2.d002.s062.t009","gold_sense_keys":["more%3:00:01::"],"candidates":[{"index":1,"sense_key":"more%3:00:01::","synset_id":"more.a.01"},{"index":2,"sense_key":"more%3:00:02::","synset_id":"more.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%3:00:01::","call_ids":["senseval2.d002.s062.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.005815000000000001,"input_uncached_usd":0.0023650000000000003,"input_cached_usd":0.0,"output_usd":0.00345,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9261422919807956}
{"item_id":"senseval2.d002.s062.t011","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s062.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":32,"reasoning_output_tokens":17},"cost":{"currency":"USD","total_usd":0.00312,"input_uncached_usd":0.00216,"input_cached_usd":0.0,"output_usd":0.00096,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2715261250268668}
{"item_id":"senseval2.d002.s063.t000","gold_sense_keys":["mystery%1:09:00::"],"candidates":[{"index":1,"sense_key":"mystery%1:10:00::","synset_id":"mystery.n.02"},{"index":2,"sense_key":"mystery%1:09:00::","synset_id":"mystery.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mystery%1:09:00::","call_ids":["senseval2.d002.s063.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mystery%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.004005,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.968581374967471}
{"item_id":"senseval2.d002.s063.t002","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":2,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"},{"index":3,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":4,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fact%1:09:01::","call_ids":["senseval2.d002.s063.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fact%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018445,"input_uncached_usd":0.002605,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.728395166923292}
{"item_id":"senseval2.d002.s063.t003","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s063.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":411,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.004725,"input_uncached_usd":0.002055,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9609924579272047}
{"item_id":"senseval2.d002.s063.t004","gold_sense_keys":["expenditure%1:21:00::"],"candidates":[{"index":1,"sense_key":"expenditure%1:04:01::","synset_id":"consumption.n.04"},{"index":2,"sense_key":"expenditure%1:21:00::","synset_id":"outgo.n.01"},{"index":3,"sense_key":"expenditure%1:04:00::","synset_id":"expending.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"expenditure%1:21:00::","call_ids":["senseval2.d002.s063.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"expenditure%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.004395,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5038933330215514}
{"item_id":"senseval2.d002.s063.t005","gold_sense_keys":["real%3:00:01::"],"candidates":[{"index":1,"sense_key":"real%5:00:02:realistic:00","synset_id":"veridical.s.01"},{"index":2,"sense_key":"real%3:00:01::","synset_id":"real.a.06"},{"index":3,"sense_key":"real%3:00:02::","synset_id":"real.a.02"},{"index":4,"sense_key":"real%5:00:00:concrete:00","synset_id":"real.s.02"},{"index":5,"sense_key":"real%3:00:03::","synset_id":"substantial.a.03"},{"index":6,"sense_key":"real%5:00:00:true:00","synset_id":"actual.s.02"},{"index":7,"sense_key":"real%5:00:00:serious:00","synset_id":"real.s.01"},{"index":8,"sense_key":"real%5:00:00:tangible:02","synset_id":"real.s.04"},{"index":9,"sense_key":"real%3:00:00::","synset_id":"real.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"real%3:00:01::","call_ids":["senseval2.d002.s063.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"real%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":788,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.00562,"input_uncached_usd":0.00394,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6579871660796925}
{"item_id":"senseval2.d002.s063.t006","gold_sense_keys":["increase%2:30:00::"],"candidates":[{"index":1,"sense_key":"increase%2:30:00::","synset_id":"increase.v.01"},{"index":2,"sense_key":"increase%2:30:02::","synset_id":"increase.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"increase%2:30:00::","call_ids":["senseval2.d002.s063.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"increase%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.00438,"input_uncached_usd":0.0021000000000000003,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4468109169974923}
{"item_id":"senseval2.d002.s063.t007","gold_sense_keys":["sharply%4:02:03::"],"candidates":[{"index":1,"sense_key":"sharply%4:02:00::","synset_id":"aggressively.r.01"},{"index":2,"sense_key":"sharply%4:02:02::","synset_id":"sharply.r.03"},{"index":3,"sense_key":"sharply%4:02:01::","synset_id":"sharply.r.02"},{"index":4,"sense_key":"sharply%4:02:03::","synset_id":"precipitously.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"sharply%4:02:03::","call_ids":["senseval2.d002.s063.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"sharply%4:02:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.00447,"input_uncached_usd":0.0026100000000000003,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.344215292017907}
{"item_id":"senseval2.d002.s063.t008","gold_sense_keys":["past%3:00:00::"],"candidates":[{"index":1,"sense_key":"past%5:00:00:outgoing:00","synset_id":"past.s.01"},{"index":2,"sense_key":"past%3:00:00::","synset_id":"past.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"past%3:00:00::","call_ids":["senseval2.d002.s063.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"past%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.004200000000000001,"input_uncached_usd":0.00216,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.733873459044844}
{"item_id":"senseval2.d002.s063.t009","gold_sense_keys":["now%4:02:00::","now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":2,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":3,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":4,"sense_key":"now%4:02:06::","synset_id":"now.r.07"},{"index":5,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":6,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":7,"sense_key":"now%4:02:07::","synset_id":"now.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"now%4:02:00::","call_ids":["senseval2.d002.s063.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"now%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":656,"cached_input_tokens":0,"output_tokens":388,"reasoning_output_tokens":372},"cost":{"currency":"USD","total_usd":0.01492,"input_uncached_usd":0.0032800000000000004,"input_cached_usd":0.0,"output_usd":0.011640000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.761112042004243}
{"item_id":"senseval2.d002.s063.t010","gold_sense_keys":["spend%2:40:00::"],"candidates":[{"index":1,"sense_key":"spend%2:42:00::","synset_id":"spend.v.01"},{"index":2,"sense_key":"spend%2:40:00::","synset_id":"spend.v.02"},{"index":3,"sense_key":"spend%2:40:01::","synset_id":"spend.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"spend%2:40:00::","call_ids":["senseval2.d002.s063.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"spend%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.004205,"input_uncached_usd":0.002255,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4256717080716044}
{"item_id":"senseval2.d002.s063.t011","gold_sense_keys":["more%4:02:01::"],"candidates":[{"index":1,"sense_key":"more%4:02:00::","synset_id":"more.r.01"},{"index":2,"sense_key":"more%4:02:01::","synset_id":"more.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"more%4:02:01::","call_ids":["senseval2.d002.s063.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"more%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":97},"cost":{"currency":"USD","total_usd":0.005520000000000001,"input_uncached_usd":0.00216,"input_cached_usd":0.0,"output_usd":0.00336,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5307148749707267}
{"item_id":"senseval2.d002.s063.t012","gold_sense_keys":["pupil%1:18:00::","pupil%1:18:01::"],"candidates":[{"index":1,"sense_key":"pupil%1:18:01::","synset_id":"schoolchild.n.01"},{"index":2,"sense_key":"pupil%1:08:00::","synset_id":"pupil.n.02"},{"index":3,"sense_key":"pupil%1:18:00::","synset_id":"student.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pupil%1:18:01::","call_ids":["senseval2.d002.s063.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pupil%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018105,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.26077729195822}
{"item_id":"senseval2.d002.s063.t013","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":2,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":3,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":4,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d002.s063.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.004975,"input_uncached_usd":0.002605,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.028605915955268}
{"item_id":"senseval2.d002.s063.t014","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":2,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":3,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"country%1:14:00::","call_ids":["senseval2.d002.s063.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01882,"input_uncached_usd":0.0029800000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.100598415941931}
{"item_id":"senseval2.d002.s063.t015","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s063.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":411,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":86},"cost":{"currency":"USD","total_usd":0.005115,"input_uncached_usd":0.002055,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.279472415917553}
{"item_id":"senseval2.d002.s063.t016","gold_sense_keys":["performance%1:04:00::","performance%1:04:03::"],"candidates":[{"index":1,"sense_key":"performance%1:04:03::","synset_id":"performance.n.04"},{"index":2,"sense_key":"performance%1:10:00::","synset_id":"performance.n.01"},{"index":3,"sense_key":"performance%1:22:00::","synset_id":"operation.n.08"},{"index":4,"sense_key":"performance%1:04:01::","synset_id":"performance.n.02"},{"index":5,"sense_key":"performance%1:04:00::","synset_id":"performance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"performance%1:04:00::","call_ids":["senseval2.d002.s063.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"performance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01883,"input_uncached_usd":0.00299,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.532793499995023}
{"item_id":"senseval2.d002.s063.t017","gold_sense_keys":["decline%2:30:01::"],"candidates":[{"index":1,"sense_key":"decline%2:30:00::","synset_id":"decline.v.04"},{"index":2,"sense_key":"decline%2:38:03::","synset_id":"decline.v.06"},{"index":3,"sense_key":"decline%2:32:01::","synset_id":"decline.v.07"},{"index":4,"sense_key":"decline%2:40:00::","synset_id":"refuse.v.02"},{"index":5,"sense_key":"decline%2:32:00::","synset_id":"refuse.v.01"},{"index":6,"sense_key":"decline%2:38:00::","synset_id":"decline.v.05"},{"index":7,"sense_key":"decline%2:30:01::","synset_id":"worsen.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"decline%2:30:01::","call_ids":["senseval2.d002.s063.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"decline%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":116},"cost":{"currency":"USD","total_usd":0.007025,"input_uncached_usd":0.0030650000000000004,"input_cached_usd":0.0,"output_usd":0.00396,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.327416874933988}
{"item_id":"senseval2.d002.s064.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":2,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":3,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":4,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":5,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":6,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":7,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:21:00::","synset_id":"way.n.12"},{"index":10,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":11,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":12,"sense_key":"way%1:07:02::","synset_id":"way.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"way%1:07:01::","call_ids":["senseval2.d002.s064.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":794,"cached_input_tokens":0,"output_tokens":212,"reasoning_output_tokens":196},"cost":{"currency":"USD","total_usd":0.01033,"input_uncached_usd":0.0039700000000000004,"input_cached_usd":0.0,"output_usd":0.00636,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.405289332964458}
{"item_id":"senseval2.d002.s065.t000","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"},{"index":2,"sense_key":"student%1:18:00::","synset_id":"student.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s065.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.0035150000000000003,"input_uncached_usd":0.0020450000000000004,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5678993330802768}
{"item_id":"senseval2.d002.s065.t001","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s065.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":392,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.00472,"input_uncached_usd":0.0019600000000000004,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.020724624977447}
{"item_id":"senseval2.d002.s065.t002","gold_sense_keys":["academic%3:01:00::"],"candidates":[{"index":1,"sense_key":"academic%5:00:00:scholarly:00","synset_id":"academic.s.02"},{"index":2,"sense_key":"academic%3:01:00::","synset_id":"academic.a.01"},{"index":3,"sense_key":"academic%5:00:00:theoretical:01","synset_id":"academic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"academic%3:01:00::","call_ids":["senseval2.d002.s065.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"academic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.00525,"input_uncached_usd":0.0022500000000000003,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9445822080597281}
{"item_id":"senseval2.d002.s065.t003","gold_sense_keys":["potential%1:26:00::"],"candidates":[{"index":1,"sense_key":"potential%1:19:00::","synset_id":"electric_potential.n.01"},{"index":2,"sense_key":"potential%1:26:00::","synset_id":"potential.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"potential%1:26:00::","call_ids":["senseval2.d002.s065.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"potential%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.003455,"input_uncached_usd":0.0020150000000000003,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5577767919749022}
{"item_id":"senseval2.d002.s065.t004","gold_sense_keys":["allow%2:42:00::"],"candidates":[{"index":1,"sense_key":"allow%2:42:00::","synset_id":"leave.v.06"},{"index":2,"sense_key":"allow%2:40:00::","synset_id":"allow.v.03"},{"index":3,"sense_key":"allow%2:32:00::","synset_id":"permit.v.01"},{"index":4,"sense_key":"allow%2:42:07::","synset_id":"admit.v.05"},{"index":5,"sense_key":"allow%2:41:00::","synset_id":"let.v.01"},{"index":6,"sense_key":"allow%2:31:00::","synset_id":"allow.v.06"},{"index":7,"sense_key":"allow%2:32:06::","synset_id":"allow.v.10"},{"index":8,"sense_key":"allow%2:40:01::","synset_id":"allow.v.09"},{"index":9,"sense_key":"allow%2:31:01::","synset_id":"allow.v.04"},{"index":10,"sense_key":"allow%2:41:01::","synset_id":"give_up.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"allow%2:42:00::","call_ids":["senseval2.d002.s065.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"allow%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":832,"cached_input_tokens":0,"output_tokens":498,"reasoning_output_tokens":482},"cost":{"currency":"USD","total_usd":0.0191,"input_uncached_usd":0.0041600000000000005,"input_cached_usd":0.0,"output_usd":0.01494,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.222894500009716}
{"item_id":"senseval2.d002.s066.t000","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s066.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.0039250000000000005,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.003776374971494}
{"item_id":"senseval2.d002.s066.t001","gold_sense_keys":["permit%2:32:00::"],"candidates":[{"index":1,"sense_key":"permit%2:41:00::","synset_id":"let.v.01"},{"index":2,"sense_key":"permit%2:32:06::","synset_id":"allow.v.10"},{"index":3,"sense_key":"permit%2:32:00::","synset_id":"permit.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"permit%2:32:00::","call_ids":["senseval2.d002.s066.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"permit%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018465,"input_uncached_usd":0.002625,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.868977917009033}
{"item_id":"senseval2.d002.s066.t002","gold_sense_keys":["graduate%2:40:01::"],"candidates":[{"index":1,"sense_key":"graduate%2:30:00::","synset_id":"calibrate.v.01"},{"index":2,"sense_key":"graduate%2:40:01::","synset_id":"graduate.v.02"},{"index":3,"sense_key":"graduate%2:40:00::","synset_id":"graduate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"graduate%2:40:01::","call_ids":["senseval2.d002.s066.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"graduate%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":126},"cost":{"currency":"USD","total_usd":0.00664,"input_uncached_usd":0.00238,"input_cached_usd":0.0,"output_usd":0.00426,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2994421249022707}
{"item_id":"senseval2.d002.s066.t004","gold_sense_keys":["master%2:31:01::"],"candidates":[{"index":1,"sense_key":"master%2:41:00::","synset_id":"dominate.v.03"},{"index":2,"sense_key":"master%2:31:00::","synset_id":"master.v.04"},{"index":3,"sense_key":"master%2:33:00::","synset_id":"overcome.v.02"},{"index":4,"sense_key":"master%2:31:01::","synset_id":"master.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"master%2:31:01::","call_ids":["senseval2.d002.s066.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"master%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.00487,"input_uncached_usd":0.00265,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0314795420272276}
{"item_id":"senseval2.d002.s066.t005","gold_sense_keys":["level%1:07:00::"],"candidates":[{"index":1,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":2,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":3,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":4,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":5,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":6,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":7,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"},{"index":8,"sense_key":"level%1:09:00::","synset_id":"level.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"level%1:07:00::","call_ids":["senseval2.d002.s066.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"level%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":718,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01943,"input_uncached_usd":0.0035900000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.070558958104812}
{"item_id":"senseval2.d002.s066.t006","gold_sense_keys":["prevail%2:42:01::"],"candidates":[{"index":1,"sense_key":"prevail%2:42:01::","synset_id":"prevail.v.02"},{"index":2,"sense_key":"prevail%2:33:00::","synset_id":"prevail.v.04"},{"index":3,"sense_key":"prevail%2:42:02::","synset_id":"predominate.v.01"},{"index":4,"sense_key":"prevail%2:32:02::","synset_id":"prevail.v.05"},{"index":5,"sense_key":"prevail%2:42:00::","synset_id":"prevail.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"prevail%2:42:01::","call_ids":["senseval2.d002.s066.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"prevail%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018670000000000003,"input_uncached_usd":0.00286,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.82956608303357}
{"item_id":"senseval2.d002.s066.t007","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:14:00::","synset_id":"class.n.06"},{"index":2,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":3,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":4,"sense_key":"year%1:28:01::","synset_id":"year.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d002.s066.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.004620000000000001,"input_uncached_usd":0.0027600000000000003,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5582780840341002}
{"item_id":"senseval2.d002.s067.t000","gold_sense_keys":["mean%2:42:00::"],"candidates":[{"index":1,"sense_key":"mean%2:32:03::","synset_id":"mean.v.03"},{"index":2,"sense_key":"mean%2:42:00::","synset_id":"entail.v.01"},{"index":3,"sense_key":"mean%2:32:01::","synset_id":"mean.v.01"},{"index":4,"sense_key":"mean%2:31:01::","synset_id":"think_of.v.04"},{"index":5,"sense_key":"mean%2:31:00::","synset_id":"intend.v.01"},{"index":6,"sense_key":"mean%2:42:03::","synset_id":"mean.v.05"},{"index":7,"sense_key":"mean%2:31:05::","synset_id":"mean.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mean%2:42:00::","call_ids":["senseval2.d002.s067.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mean%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":131},"cost":{"currency":"USD","total_usd":0.007615,"input_uncached_usd":0.0032350000000000005,"input_cached_usd":0.0,"output_usd":0.00438,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3630219160113484}
{"item_id":"senseval2.d002.s067.t001","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":2,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":3,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["senseval2.d002.s067.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.004065000000000001,"input_uncached_usd":0.0023850000000000004,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6401088340207934}
{"item_id":"senseval2.d002.s067.t002","gold_sense_keys":["purpose%1:07:00::","purpose%1:09:00::"],"candidates":[{"index":1,"sense_key":"purpose%1:07:00::","synset_id":"function.n.02"},{"index":2,"sense_key":"purpose%1:09:00::","synset_id":"purpose.n.01"},{"index":3,"sense_key":"purpose%1:07:01::","synset_id":"determination.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"purpose%1:09:00::","call_ids":["senseval2.d002.s067.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"purpose%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":195,"reasoning_output_tokens":179},"cost":{"currency":"USD","total_usd":0.008325,"input_uncached_usd":0.002475,"input_cached_usd":0.0,"output_usd":0.00585,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.756352207972668}
{"item_id":"senseval2.d002.s067.t004","gold_sense_keys":["permit%2:41:00::"],"candidates":[{"index":1,"sense_key":"permit%2:41:00::","synset_id":"let.v.01"},{"index":2,"sense_key":"permit%2:32:06::","synset_id":"allow.v.10"},{"index":3,"sense_key":"permit%2:32:00::","synset_id":"permit.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"permit%2:41:00::","call_ids":["senseval2.d002.s067.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"permit%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.0060550000000000005,"input_uncached_usd":0.0025150000000000003,"input_cached_usd":0.0,"output_usd":0.00354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9839838329935446}
{"item_id":"senseval2.d002.s067.t007","gold_sense_keys":["flourish%2:40:00::"],"candidates":[{"index":1,"sense_key":"flourish%2:35:00::","synset_id":"brandish.v.01"},{"index":2,"sense_key":"flourish%2:30:01::","synset_id":"boom.v.05"},{"index":3,"sense_key":"flourish%2:40:00::","synset_id":"thrive.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"flourish%2:40:00::","call_ids":["senseval2.d002.s067.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"flourish%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.0058449999999999995,"input_uncached_usd":0.002395,"input_cached_usd":0.0,"output_usd":0.00345,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.779942875029519}
{"item_id":"senseval2.d002.s067.t008","gold_sense_keys":["clearly%4:02:00::"],"candidates":[{"index":1,"sense_key":"clearly%4:02:00::","synset_id":"clearly.r.01"},{"index":2,"sense_key":"clearly%4:02:03::","synset_id":"clearly.r.04"},{"index":3,"sense_key":"clearly%4:02:02::","synset_id":"distinctly.r.01"},{"index":4,"sense_key":"clearly%4:02:01::","synset_id":"intelligibly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"clearly%4:02:00::","call_ids":["senseval2.d002.s067.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"clearly%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":507,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.005325,"input_uncached_usd":0.0025350000000000004,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":104.29404624993913}
{"item_id":"senseval2.d002.s067.t009","gold_sense_keys":["desirable%3:00:00::"],"candidates":[{"index":1,"sense_key":"desirable%3:00:00::","synset_id":"desirable.a.01"},{"index":2,"sense_key":"desirable%5:00:00:eligible:00","synset_id":"desirable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"desirable%3:00:00::","call_ids":["senseval2.d002.s067.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"desirable%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.003915,"input_uncached_usd":0.002085,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7062972499988973}
{"item_id":"senseval2.d002.s067.t010","gold_sense_keys":["less%4:02:00::"],"candidates":[{"index":1,"sense_key":"less%4:02:00::","synset_id":"less.r.01"},{"index":2,"sense_key":"less%4:02:01::","synset_id":"less.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"less%4:02:00::","call_ids":["senseval2.d002.s067.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"less%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.00468,"input_uncached_usd":0.0020700000000000002,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7774680000729859}
{"item_id":"senseval2.d002.s067.t012","gold_sense_keys":["get%2:40:00::"],"candidates":[{"index":1,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":2,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":3,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":4,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":5,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":6,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":7,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":8,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":9,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":10,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":11,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":12,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":13,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":14,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"},{"index":15,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":16,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":17,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":18,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":19,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":20,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":21,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":22,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":23,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":24,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":25,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":26,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":27,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":28,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":29,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":30,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":31,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":32,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":33,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":34,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":35,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":36,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":24,"chosen_sense_key":"get%2:40:00::","call_ids":["senseval2.d002.s067.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":24,"predicted_sense_key":"get%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1968,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02568,"input_uncached_usd":0.009840000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.633726791013032}
{"item_id":"senseval2.d002.s067.t013","gold_sense_keys":["necessary%3:00:00::"],"candidates":[{"index":1,"sense_key":"necessary%3:00:00::","synset_id":"necessary.a.01"},{"index":2,"sense_key":"necessary%5:00:00:inevitable:00","synset_id":"necessary.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"necessary%3:00:00::","call_ids":["senseval2.d002.s067.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"necessary%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":393,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.0037050000000000004,"input_uncached_usd":0.001965,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8420484999660403}
{"item_id":"senseval2.d002.s067.t014","gold_sense_keys":["study%1:09:03::"],"candidates":[{"index":1,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":2,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":3,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":4,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":5,"sense_key":"study%1:10:01::","synset_id":"study.n.10"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":8,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":9,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":10,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"study%1:09:03::","call_ids":["senseval2.d002.s067.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"study%1:09:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.006535000000000001,"input_uncached_usd":0.0037450000000000005,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1973081250907853}
{"item_id":"senseval2.d002.s067.t015","gold_sense_keys":["enter%2:33:00::","enter%2:38:00::","enter%2:42:00::"],"candidates":[{"index":1,"sense_key":"enter%2:35:03::","synset_id":"insert.v.01"},{"index":2,"sense_key":"enter%2:36:00::","synset_id":"enter.v.06"},{"index":3,"sense_key":"enter%2:32:04::","synset_id":"record.v.01"},{"index":4,"sense_key":"enter%2:38:00::","synset_id":"enter.v.01"},{"index":5,"sense_key":"enter%2:41:06::","synset_id":"enroll.v.01"},{"index":6,"sense_key":"enter%2:30:10::","synset_id":"embark.v.02"},{"index":7,"sense_key":"enter%2:33:00::","synset_id":"enter.v.02"},{"index":8,"sense_key":"enter%2:41:00::","synset_id":"accede.v.02"},{"index":9,"sense_key":"enter%2:42:00::","synset_id":"figure.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"enter%2:33:00::","call_ids":["senseval2.d002.s067.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"enter%2:33:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":9737,"reasoning_output_tokens":9613},"cost":{"currency":"USD","total_usd":0.295595,"input_uncached_usd":0.0034850000000000003,"input_cached_usd":0.0,"output_usd":0.29211,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":211.7377185409423}
{"item_id":"senseval2.d002.s067.t017","gold_sense_keys":["world%1:14:01::"],"candidates":[{"index":1,"sense_key":"world%1:17:00::","synset_id":"earth.n.01"},{"index":2,"sense_key":"world%1:14:02::","synset_id":"populace.n.01"},{"index":3,"sense_key":"world%1:17:02::","synset_id":"world.n.06"},{"index":4,"sense_key":"world%1:09:00::","synset_id":"world.n.03"},{"index":5,"sense_key":"world%1:14:01::","synset_id":"world.n.02"},{"index":6,"sense_key":"world%1:05:00::","synset_id":"world.n.08"},{"index":7,"sense_key":"world%1:09:01::","synset_id":"worldly_concern.n.01"},{"index":8,"sense_key":"world%1:17:01::","synset_id":"universe.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"world%1:17:02::","call_ids":["senseval2.d002.s067.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"world%1:17:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":731,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019495,"input_uncached_usd":0.0036550000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.588761750026606}
{"item_id":"senseval2.d002.s067.t018","gold_sense_keys":["work%1:04:01::"],"candidates":[{"index":1,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":2,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":3,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":4,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":5,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"},{"index":6,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":7,"sense_key":"work%1:09:00::","synset_id":"study.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"work%1:04:01::","call_ids":["senseval2.d002.s067.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"work%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.006265,"input_uncached_usd":0.0034750000000000002,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0483488750178367}
{"item_id":"senseval2.d002.s068.t000","gold_sense_keys":["notion%1:09:02::"],"candidates":[{"index":1,"sense_key":"notion%1:09:01::","synset_id":"notion.n.03"},{"index":2,"sense_key":"notion%1:06:00::","synset_id":"notion.n.04"},{"index":3,"sense_key":"notion%1:09:02::","synset_id":"impression.n.01"},{"index":4,"sense_key":"notion%1:09:00::","synset_id":"notion.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"notion%1:09:02::","call_ids":["senseval2.d002.s068.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"notion%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":99},"cost":{"currency":"USD","total_usd":0.005975,"input_uncached_usd":0.0025250000000000003,"input_cached_usd":0.0,"output_usd":0.00345,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3403614580165595}
{"item_id":"senseval2.d002.s069.t000","gold_sense_keys":["purpose%1:07:00::","purpose%1:09:00::"],"candidates":[{"index":1,"sense_key":"purpose%1:07:01::","synset_id":"determination.n.02"},{"index":2,"sense_key":"purpose%1:09:00::","synset_id":"purpose.n.01"},{"index":3,"sense_key":"purpose%1:07:00::","synset_id":"function.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"purpose%1:09:00::","call_ids":["senseval2.d002.s069.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"purpose%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":276,"reasoning_output_tokens":260},"cost":{"currency":"USD","total_usd":0.010560000000000002,"input_uncached_usd":0.0022800000000000003,"input_cached_usd":0.0,"output_usd":0.008280000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.44767129200045}
{"item_id":"senseval2.d002.s069.t001","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":2,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"},{"index":3,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":4,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":5,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":6,"sense_key":"education%1:09:01::","synset_id":"education.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s069.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018655,"input_uncached_usd":0.0028150000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.110071541042998}
{"item_id":"senseval2.d002.s069.t002","gold_sense_keys":["encourage%2:32:00::","encourage%2:37:00::"],"candidates":[{"index":1,"sense_key":"encourage%2:41:00::","synset_id":"promote.v.01"},{"index":2,"sense_key":"encourage%2:37:00::","synset_id":"encourage.v.02"},{"index":3,"sense_key":"encourage%2:32:00::","synset_id":"encourage.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"encourage%2:32:00::","call_ids":["senseval2.d002.s069.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"encourage%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017845,"input_uncached_usd":0.0020350000000000004,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.7278317080345}
{"item_id":"senseval2.d002.s069.t003","gold_sense_keys":["young%3:00:00::"],"candidates":[{"index":1,"sense_key":"young%5:00:00:inexperienced:00","synset_id":"unseasoned.s.02"},{"index":2,"sense_key":"young%5:00:00:new:00","synset_id":"young.s.03"},{"index":3,"sense_key":"young%5:00:01:young:00","synset_id":"youthful.s.01"},{"index":4,"sense_key":"young%3:00:00::","synset_id":"young.a.01"},{"index":5,"sense_key":"young%5:00:00:early:02","synset_id":"new.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"young%3:00:00::","call_ids":["senseval2.d002.s069.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"young%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.0043300000000000005,"input_uncached_usd":0.0025900000000000003,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3735588339623064}
{"item_id":"senseval2.d002.s069.t004","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":2,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":5,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":6,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":7,"sense_key":"man%1:05:00::","synset_id":"world.n.08"},{"index":8,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":9,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":10,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":11,"sense_key":"man%1:18:08::","synset_id":"man.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval2.d002.s069.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":833,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":136},"cost":{"currency":"USD","total_usd":0.008725,"input_uncached_usd":0.004165,"input_cached_usd":0.0,"output_usd":0.00456,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.024007000029087}
{"item_id":"senseval2.d002.s069.t005","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"},{"index":2,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d002.s069.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":294,"reasoning_output_tokens":278},"cost":{"currency":"USD","total_usd":0.01129,"input_uncached_usd":0.0024700000000000004,"input_cached_usd":0.0,"output_usd":0.00882,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.2068310000468045}
{"item_id":"senseval2.d002.s069.t006","gold_sense_keys":["realize%2:36:00::"],"candidates":[{"index":1,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":2,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"},{"index":3,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"},{"index":4,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"},{"index":5,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"},{"index":6,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"realize%2:36:00::","call_ids":["senseval2.d002.s069.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"realize%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.00524,"input_uncached_usd":0.0029600000000000004,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6599456670228392}
{"item_id":"senseval2.d002.s069.t007","gold_sense_keys":["full%5:00:00:complete:00","full%5:00:01:whole:00"],"candidates":[{"index":1,"sense_key":"full%5:00:00:high:02","synset_id":"broad.s.04"},{"index":2,"sense_key":"full%5:00:01:whole:00","synset_id":"entire.s.01"},{"index":3,"sense_key":"full%5:00:00:complete:00","synset_id":"full.s.02"},{"index":4,"sense_key":"full%3:00:00::","synset_id":"full.a.01"},{"index":5,"sense_key":"full%3:00:01::","synset_id":"full.a.05"},{"index":6,"sense_key":"full%5:00:00:nourished:00","synset_id":"full.s.03"},{"index":7,"sense_key":"full%5:00:02:ample:00","synset_id":"wide.s.04"},{"index":8,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"full%5:00:01:whole:00","call_ids":["senseval2.d002.s069.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"full%5:00:01:whole:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018945,"input_uncached_usd":0.003105,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.709219874930568}
{"item_id":"senseval2.d002.s069.t008","gold_sense_keys":["academic%3:01:00::"],"candidates":[{"index":1,"sense_key":"academic%5:00:00:theoretical:01","synset_id":"academic.s.01"},{"index":2,"sense_key":"academic%5:00:00:scholarly:00","synset_id":"academic.s.02"},{"index":3,"sense_key":"academic%3:01:00::","synset_id":"academic.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"academic%3:01:00::","call_ids":["senseval2.d002.s069.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"academic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.004280000000000001,"input_uncached_usd":0.0020900000000000003,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8822530830511823}
{"item_id":"senseval2.d002.s069.t009","gold_sense_keys":["potential%1:26:00::"],"candidates":[{"index":1,"sense_key":"potential%1:19:00::","synset_id":"electric_potential.n.01"},{"index":2,"sense_key":"potential%1:26:00::","synset_id":"potential.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"potential%1:26:00::","call_ids":["senseval2.d002.s069.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"potential%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.00326,"input_uncached_usd":0.00185,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5213449170114473}
{"item_id":"senseval2.d002.s070.t000","gold_sense_keys":["actually%4:02:01::"],"candidates":[{"index":1,"sense_key":"actually%4:02:03::","synset_id":"actually.r.03"},{"index":2,"sense_key":"actually%4:02:01::","synset_id":"actually.r.01"},{"index":3,"sense_key":"actually%4:02:00::","synset_id":"actually.r.02"},{"index":4,"sense_key":"actually%4:02:02::","synset_id":"actually.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"actually%4:02:01::","call_ids":["senseval2.d002.s070.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"actually%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":114},"cost":{"currency":"USD","total_usd":0.006370000000000001,"input_uncached_usd":0.0024700000000000004,"input_cached_usd":0.0,"output_usd":0.0039000000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.048665708047338}
{"item_id":"senseval2.d002.s070.t001","gold_sense_keys":["believe%2:31:00::"],"candidates":[{"index":1,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":2,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"},{"index":3,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"},{"index":4,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"},{"index":5,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"believe%2:31:00::","call_ids":["senseval2.d002.s070.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"believe%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":1564,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.049585000000000004,"input_uncached_usd":0.0026650000000000003,"input_cached_usd":0.0,"output_usd":0.04692,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":30.691810582997277}
{"item_id":"senseval2.d002.s070.t002","gold_sense_keys":["equal%3:00:00::"],"candidates":[{"index":1,"sense_key":"equal%3:00:03::","synset_id":"adequate.a.01"},{"index":2,"sense_key":"equal%3:00:00::","synset_id":"equal.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"equal%3:00:00::","call_ids":["senseval2.d002.s070.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"equal%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":399,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0036750000000000003,"input_uncached_usd":0.0019950000000000002,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.514312749961391}
{"item_id":"senseval2.d002.s070.t003","gold_sense_keys":["academic%3:01:00::"],"candidates":[{"index":1,"sense_key":"academic%5:00:00:theoretical:01","synset_id":"academic.s.01"},{"index":2,"sense_key":"academic%3:01:00::","synset_id":"academic.a.01"},{"index":3,"sense_key":"academic%5:00:00:scholarly:00","synset_id":"academic.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"academic%3:01:00::","call_ids":["senseval2.d002.s070.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"academic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.004485,"input_uncached_usd":0.0021750000000000003,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.215596167021431}
{"item_id":"senseval2.d002.s070.t004","gold_sense_keys":["potential%1:26:00::"],"candidates":[{"index":1,"sense_key":"potential%1:19:00::","synset_id":"electric_potential.n.01"},{"index":2,"sense_key":"potential%1:26:00::","synset_id":"potential.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"potential%1:26:00::","call_ids":["senseval2.d002.s070.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"potential%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":39,"reasoning_output_tokens":24},"cost":{"currency":"USD","total_usd":0.003105,"input_uncached_usd":0.001935,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8660854999907315}
{"item_id":"senseval2.d002.s071.t000","gold_sense_keys":["generally%4:02:02::"],"candidates":[{"index":1,"sense_key":"generally%4:02:01::","synset_id":"generally.r.02"},{"index":2,"sense_key":"generally%4:02:00::","synset_id":"broadly.r.01"},{"index":3,"sense_key":"generally%4:02:02::","synset_id":"by_and_large.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"generally%4:02:02::","call_ids":["senseval2.d002.s071.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"generally%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0181,"input_uncached_usd":0.0022600000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.2670198330888525}
{"item_id":"senseval2.d002.s071.t001","gold_sense_keys":["desirable%3:00:00::"],"candidates":[{"index":1,"sense_key":"desirable%5:00:00:eligible:00","synset_id":"desirable.s.01"},{"index":2,"sense_key":"desirable%3:00:00::","synset_id":"desirable.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"desirable%3:00:00::","call_ids":["senseval2.d002.s071.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"desirable%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":401,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.003775,"input_uncached_usd":0.0020050000000000003,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6268937080167234}
{"item_id":"senseval2.d002.s071.t002","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":2,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":3,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":4,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":5,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d002.s071.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":600,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00483,"input_uncached_usd":0.003,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.626748290960677}
{"item_id":"senseval2.d002.s071.t005","gold_sense_keys":["rather%4:02:02::"],"candidates":[{"index":1,"sense_key":"rather%4:02:04::","synset_id":"rather.r.02"},{"index":2,"sense_key":"rather%4:02:02::","synset_id":"rather.r.01"},{"index":3,"sense_key":"rather%4:02:00::","synset_id":"quite.r.01"},{"index":4,"sense_key":"rather%4:02:01::","synset_id":"preferably.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"rather%4:02:02::","call_ids":["senseval2.d002.s071.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"rather%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":691,"reasoning_output_tokens":670},"cost":{"currency":"USD","total_usd":0.02326,"input_uncached_usd":0.00253,"input_cached_usd":0.0,"output_usd":0.020730000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.363965332973748}
{"item_id":"senseval2.d002.s071.t006","gold_sense_keys":["new%3:00:00::"],"candidates":[{"index":1,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":2,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":3,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":4,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":5,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":6,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":7,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":8,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"},{"index":9,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":10,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":11,"sense_key":"new%3:00:00::","synset_id":"new.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d002.s071.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":220,"reasoning_output_tokens":204},"cost":{"currency":"USD","total_usd":0.010615000000000001,"input_uncached_usd":0.004015,"input_cached_usd":0.0,"output_usd":0.0066,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.433834499912336}
{"item_id":"senseval2.d002.s071.t007","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval2.d002.s071.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.004240000000000001,"input_uncached_usd":0.0020800000000000003,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.45441829203628}
{"item_id":"senseval2.d002.s072.t003","gold_sense_keys":["often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:02::","synset_id":"often.r.03"},{"index":2,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":3,"sense_key":"often%4:02:01::","synset_id":"much.r.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"often%4:02:02::","call_ids":["senseval2.d002.s072.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"often%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.00457,"input_uncached_usd":0.0020800000000000003,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.665264374925755}
{"item_id":"senseval2.d002.s073.t000","gold_sense_keys":["dubious%5:00:00:questionable:00"],"candidates":[{"index":1,"sense_key":"dubious%5:00:00:unconvinced:00","synset_id":"dubious.s.03"},{"index":2,"sense_key":"dubious%5:00:00:uncertain:02","synset_id":"doubtful.s.02"},{"index":3,"sense_key":"dubious%5:00:00:questionable:00","synset_id":"doubtful.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"dubious%5:00:00:questionable:00","call_ids":["senseval2.d002.s073.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"dubious%5:00:00:questionable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004295,"input_uncached_usd":0.002255,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5938932910794392}
{"item_id":"senseval2.d002.s073.t001","gold_sense_keys":["psychological%3:01:00::"],"candidates":[{"index":1,"sense_key":"psychological%5:00:00:mental:00","synset_id":"psychological.s.01"},{"index":2,"sense_key":"psychological%3:01:00::","synset_id":"psychological.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"psychological%3:01:00::","call_ids":["senseval2.d002.s073.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"psychological%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.00404,"input_uncached_usd":0.00185,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5671749589964747}
{"item_id":"senseval2.d002.s073.t004","gold_sense_keys":["rather%4:02:02::"],"candidates":[{"index":1,"sense_key":"rather%4:02:04::","synset_id":"rather.r.02"},{"index":2,"sense_key":"rather%4:02:00::","synset_id":"quite.r.01"},{"index":3,"sense_key":"rather%4:02:01::","synset_id":"preferably.r.01"},{"index":4,"sense_key":"rather%4:02:02::","synset_id":"rather.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"rather%4:02:02::","call_ids":["senseval2.d002.s073.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"rather%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":142},"cost":{"currency":"USD","total_usd":0.0071400000000000005,"input_uncached_usd":0.0024300000000000003,"input_cached_usd":0.0,"output_usd":0.00471,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.505517791956663}
{"item_id":"senseval2.d002.s073.t005","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s073.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":362,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.00406,"input_uncached_usd":0.0018100000000000002,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8089609999442473}
{"item_id":"senseval2.d002.s073.t006","gold_sense_keys":["experience%1:09:01::"],"candidates":[{"index":1,"sense_key":"experience%1:09:01::","synset_id":"experience.n.01"},{"index":2,"sense_key":"experience%1:11:00::","synset_id":"experience.n.03"},{"index":3,"sense_key":"experience%1:09:00::","synset_id":"experience.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"experience%1:09:01::","call_ids":["senseval2.d002.s073.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"experience%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.0046949999999999995,"input_uncached_usd":0.002085,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.703216083929874}
{"item_id":"senseval2.d002.s074.t000","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::"],"candidates":[{"index":1,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":2,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":3,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":4,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":5,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":6,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reason%1:10:01::","call_ids":["senseval2.d002.s074.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reason%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":196,"reasoning_output_tokens":180},"cost":{"currency":"USD","total_usd":0.008980000000000002,"input_uncached_usd":0.0031000000000000003,"input_cached_usd":0.0,"output_usd":0.00588,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.31133816705551}
{"item_id":"senseval2.d002.s074.t001","gold_sense_keys":["american%3:01:01::"],"candidates":[{"index":1,"sense_key":"american%3:01:01::","synset_id":"american.a.01"},{"index":2,"sense_key":"american%3:01:00::","synset_id":"american.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%3:01:01::","call_ids":["senseval2.d002.s074.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":421,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.0038450000000000003,"input_uncached_usd":0.002105,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4422186249867082}
{"item_id":"senseval2.d002.s074.t002","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s074.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.0040149999999999995,"input_uncached_usd":0.002095,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9883642909117043}
{"item_id":"senseval2.d002.s074.t003","gold_sense_keys":["so%4:02:02::"],"candidates":[{"index":1,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":2,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":3,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":4,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":5,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":6,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":7,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":8,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":9,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"},{"index":10,"sense_key":"so%4:02:07::","synset_id":"so.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"so%4:02:02::","call_ids":["senseval2.d002.s074.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"so%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":821,"cached_input_tokens":0,"output_tokens":327,"reasoning_output_tokens":311},"cost":{"currency":"USD","total_usd":0.013915,"input_uncached_usd":0.004105,"input_cached_usd":0.0,"output_usd":0.009810000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.222829582984559}
{"item_id":"senseval2.d002.s074.t006","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":2,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":3,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":4,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"compare%2:31:00::","call_ids":["senseval2.d002.s074.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":248,"reasoning_output_tokens":232},"cost":{"currency":"USD","total_usd":0.009980000000000001,"input_uncached_usd":0.00254,"input_cached_usd":0.0,"output_usd":0.00744,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.796656291000545}
{"item_id":"senseval2.d002.s074.t011","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s074.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00401,"input_uncached_usd":0.0020900000000000003,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.515950790955685}
{"item_id":"senseval2.d002.s074.t012","gold_sense_keys":["influence%1:19:00::"],"candidates":[{"index":1,"sense_key":"influence%1:04:00::","synset_id":"influence.n.02"},{"index":2,"sense_key":"influence%1:18:00::","synset_id":"influence.n.05"},{"index":3,"sense_key":"influence%1:19:00::","synset_id":"influence.n.04"},{"index":4,"sense_key":"influence%1:09:00::","synset_id":"influence.n.03"},{"index":5,"sense_key":"influence%1:07:00::","synset_id":"influence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"influence%1:19:00::","call_ids":["senseval2.d002.s074.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"influence%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.005365,"input_uncached_usd":0.002635,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0026879170909524}
{"item_id":"senseval2.d002.s074.t013","gold_sense_keys":["american%3:01:01::"],"candidates":[{"index":1,"sense_key":"american%3:01:01::","synset_id":"american.a.01"},{"index":2,"sense_key":"american%3:01:00::","synset_id":"american.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%3:01:01::","call_ids":["senseval2.d002.s074.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":421,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.004295,"input_uncached_usd":0.002105,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9719937500776723}
{"item_id":"senseval2.d002.s075.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":2,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":3,"sense_key":"want%2:42:01::","synset_id":"want.v.05"},{"index":4,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":5,"sense_key":"want%2:35:00::","synset_id":"want.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"want%2:37:00::","call_ids":["senseval2.d002.s075.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004745,"input_uncached_usd":0.002735,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5904095000587404}
{"item_id":"senseval2.d002.s075.t001","gold_sense_keys":["appreciate%2:31:00::"],"candidates":[{"index":1,"sense_key":"appreciate%2:30:00::","synset_id":"appreciate.v.04"},{"index":2,"sense_key":"appreciate%2:31:00::","synset_id":"appreciate.v.02"},{"index":3,"sense_key":"appreciate%2:37:00::","synset_id":"appreciate.v.01"},{"index":4,"sense_key":"appreciate%2:40:00::","synset_id":"prize.v.01"},{"index":5,"sense_key":"appreciate%2:30:01::","synset_id":"appreciate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"appreciate%2:31:00::","call_ids":["senseval2.d002.s075.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"appreciate%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.00476,"input_uncached_usd":0.00272,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7586506250081584}
{"item_id":"senseval2.d002.s075.t002","gold_sense_keys":["just%4:02:05::"],"candidates":[{"index":1,"sense_key":"just%4:02:06::","synset_id":"just.r.06"},{"index":2,"sense_key":"just%4:02:04::","synset_id":"barely.r.01"},{"index":3,"sense_key":"just%4:02:05::","synset_id":"precisely.r.01"},{"index":4,"sense_key":"just%4:02:01::","synset_id":"just.r.03"},{"index":5,"sense_key":"just%4:02:00::","synset_id":"merely.r.01"},{"index":6,"sense_key":"just%4:02:03::","synset_id":"just.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"just%4:02:05::","call_ids":["senseval2.d002.s075.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"just%4:02:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":115},"cost":{"currency":"USD","total_usd":0.007125000000000001,"input_uncached_usd":0.0031950000000000004,"input_cached_usd":0.0,"output_usd":0.00393,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5895259169628844}
{"item_id":"senseval2.d002.s075.t004","gold_sense_keys":["situation%1:26:00::"],"candidates":[{"index":1,"sense_key":"situation%1:26:02::","synset_id":"situation.n.03"},{"index":2,"sense_key":"situation%1:26:01::","synset_id":"situation.n.02"},{"index":3,"sense_key":"situation%1:26:00::","synset_id":"situation.n.01"},{"index":4,"sense_key":"situation%1:15:00::","synset_id":"site.n.02"},{"index":5,"sense_key":"situation%1:04:00::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"situation%1:26:00::","call_ids":["senseval2.d002.s075.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"situation%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":105},"cost":{"currency":"USD","total_usd":0.006605,"input_uncached_usd":0.002975,"input_cached_usd":0.0,"output_usd":0.00363,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.901825667009689}
{"item_id":"senseval2.d002.s075.t005","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"},{"index":2,"sense_key":"student%1:18:00::","synset_id":"student.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s075.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":431,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004195000000000001,"input_uncached_usd":0.0021550000000000002,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.200747957918793}
{"item_id":"senseval2.d002.s075.t006","gold_sense_keys":["add%2:31:00::"],"candidates":[{"index":1,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"},{"index":2,"sense_key":"add%2:32:00::","synset_id":"total.v.02"},{"index":3,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":4,"sense_key":"add%2:32:01::","synset_id":"add.v.02"},{"index":5,"sense_key":"add%2:42:00::","synset_id":"add.v.06"},{"index":6,"sense_key":"add%2:31:00::","synset_id":"add.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"add%2:31:00::","call_ids":["senseval2.d002.s075.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"add%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":199,"reasoning_output_tokens":183},"cost":{"currency":"USD","total_usd":0.009115000000000002,"input_uncached_usd":0.0031450000000000002,"input_cached_usd":0.0,"output_usd":0.0059700000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5472562089562416}
{"item_id":"senseval2.d002.s075.t007","gold_sense_keys":["subtract%2:31:00::"],"candidates":[{"index":1,"sense_key":"subtract%2:30:00::","synset_id":"subtract.v.02"},{"index":2,"sense_key":"subtract%2:31:00::","synset_id":"subtract.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"subtract%2:31:00::","call_ids":["senseval2.d002.s075.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"subtract%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.0038800000000000006,"input_uncached_usd":0.0021100000000000003,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6350915000075474}
{"item_id":"senseval2.d002.s075.t008","gold_sense_keys":["learn%2:31:00::"],"candidates":[{"index":1,"sense_key":"learn%2:31:00::","synset_id":"learn.v.01"},{"index":2,"sense_key":"learn%2:31:03::","synset_id":"memorize.v.01"},{"index":3,"sense_key":"learn%2:31:01::","synset_id":"learn.v.02"},{"index":4,"sense_key":"learn%2:32:00::","synset_id":"teach.v.01"},{"index":5,"sense_key":"learn%2:31:02::","synset_id":"learn.v.04"},{"index":6,"sense_key":"learn%2:32:02::","synset_id":"determine.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"learn%2:31:00::","call_ids":["senseval2.d002.s075.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"learn%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019110000000000002,"input_uncached_usd":0.0032700000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.748818792053498}
{"item_id":"senseval2.d002.s075.t010","gold_sense_keys":["basis%1:09:00::"],"candidates":[{"index":1,"sense_key":"basis%1:24:01::","synset_id":"basis.n.03"},{"index":2,"sense_key":"basis%1:24:00::","synset_id":"footing.n.02"},{"index":3,"sense_key":"basis%1:09:00::","synset_id":"basis.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"basis%1:09:00::","call_ids":["senseval2.d002.s075.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"basis%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":94},"cost":{"currency":"USD","total_usd":0.005765,"input_uncached_usd":0.002465,"input_cached_usd":0.0,"output_usd":0.0033,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6889510420151055}
{"item_id":"senseval2.d002.s075.t011","gold_sense_keys":["mathematical%3:01:00::"],"candidates":[{"index":1,"sense_key":"mathematical%3:00:00::","synset_id":"numerical.a.03"},{"index":2,"sense_key":"mathematical%3:01:00::","synset_id":"mathematical.a.01"},{"index":3,"sense_key":"mathematical%5:00:00:exact:00","synset_id":"mathematical.s.03"},{"index":4,"sense_key":"mathematical%5:00:00:unquestionable:00","synset_id":"mathematical.s.01"},{"index":5,"sense_key":"mathematical%5:00:00:possible:00","synset_id":"mathematical.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mathematical%3:01:00::","call_ids":["senseval2.d002.s075.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mathematical%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.00477,"input_uncached_usd":0.0027600000000000003,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.27112979197409}
{"item_id":"senseval2.d002.s075.t013","gold_sense_keys":["read%2:31:00::"],"candidates":[{"index":1,"sense_key":"read%2:32:02::","synset_id":"read.v.08"},{"index":2,"sense_key":"read%2:31:01::","synset_id":"read.v.03"},{"index":3,"sense_key":"read%2:31:09::","synset_id":"take.v.06"},{"index":4,"sense_key":"read%2:36:05::","synset_id":"read.v.09"},{"index":5,"sense_key":"read%2:31:06::","synset_id":"read.v.04"},{"index":6,"sense_key":"read%2:31:05::","synset_id":"learn.v.04"},{"index":7,"sense_key":"read%2:42:00::","synset_id":"read.v.02"},{"index":8,"sense_key":"read%2:31:08::","synset_id":"read.v.10"},{"index":9,"sense_key":"read%2:31:04::","synset_id":"understand.v.03"},{"index":10,"sense_key":"read%2:31:02::","synset_id":"read.v.05"},{"index":11,"sense_key":"read%2:31:00::","synset_id":"read.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"read%2:31:00::","call_ids":["senseval2.d002.s075.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"read%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":830,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.00664,"input_uncached_usd":0.00415,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":95.61386783292983}
{"item_id":"senseval2.d002.s075.t014","gold_sense_keys":["article%1:10:00::"],"candidates":[{"index":1,"sense_key":"article%1:10:00::","synset_id":"article.n.01"},{"index":2,"sense_key":"article%1:03:00::","synset_id":"article.n.02"},{"index":3,"sense_key":"article%1:10:02::","synset_id":"article.n.03"},{"index":4,"sense_key":"article%1:10:01::","synset_id":"article.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"article%1:10:00::","call_ids":["senseval2.d002.s075.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"article%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00434,"input_uncached_usd":0.00239,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6117454170016572}
{"item_id":"senseval2.d002.s075.t015","gold_sense_keys":["recent%5:00:00:new:00"],"candidates":[{"index":1,"sense_key":"recent%5:00:00:new:00","synset_id":"recent.s.01"},{"index":2,"sense_key":"recent%5:00:00:past:00","synset_id":"late.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recent%5:00:00:new:00","call_ids":["senseval2.d002.s075.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recent%5:00:00:new:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01797,"input_uncached_usd":0.0021300000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.835839875042439}
{"item_id":"senseval2.d002.s075.t017","gold_sense_keys":["major%1:18:01::"],"candidates":[{"index":1,"sense_key":"major%1:09:00::","synset_id":"major.n.04"},{"index":2,"sense_key":"major%1:18:02::","synset_id":"major.n.02"},{"index":3,"sense_key":"major%1:18:01::","synset_id":"major.n.03"},{"index":4,"sense_key":"major%1:18:00::","synset_id":"major.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"major%1:18:01::","call_ids":["senseval2.d002.s075.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"major%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.004435,"input_uncached_usd":0.002575,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.607607249985449}
{"item_id":"senseval2.d002.s076.t000","gold_sense_keys":["most%4:02:00::"],"candidates":[{"index":1,"sense_key":"most%4:02:02::","synset_id":"about.r.07"},{"index":2,"sense_key":"most%4:02:00::","synset_id":"most.r.01"},{"index":3,"sense_key":"most%4:02:01::","synset_id":"most.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"most%4:02:00::","call_ids":["senseval2.d002.s076.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"most%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":105},"cost":{"currency":"USD","total_usd":0.00614,"input_uncached_usd":0.00251,"input_cached_usd":0.0,"output_usd":0.00363,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3724534589564428}
{"item_id":"senseval2.d002.s076.t001","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"},{"index":2,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":3,"sense_key":"important%3:00:00::","synset_id":"important.a.01"},{"index":4,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"},{"index":5,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"important%3:00:00::","call_ids":["senseval2.d002.s076.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"important%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":593,"cached_input_tokens":0,"output_tokens":262,"reasoning_output_tokens":246},"cost":{"currency":"USD","total_usd":0.010825000000000001,"input_uncached_usd":0.002965,"input_cached_usd":0.0,"output_usd":0.00786,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.516274082940072}
{"item_id":"senseval2.d002.s076.t002","gold_sense_keys":["school%1:14:00::","school%1:14:03::"],"candidates":[{"index":1,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":2,"sense_key":"school%1:09:00::","synset_id":"school.n.03"},{"index":3,"sense_key":"school%1:14:00::","synset_id":"school.n.01"},{"index":4,"sense_key":"school%1:14:01::","synset_id":"school.n.07"},{"index":5,"sense_key":"school%1:06:00::","synset_id":"school.n.02"},{"index":6,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":7,"sense_key":"school%1:28:00::","synset_id":"school.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"school%1:14:00::","call_ids":["senseval2.d002.s076.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"school%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":664,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":117},"cost":{"currency":"USD","total_usd":0.0073100000000000005,"input_uncached_usd":0.0033200000000000005,"input_cached_usd":0.0,"output_usd":0.0039900000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.765381791978143}
{"item_id":"senseval2.d002.s076.t003","gold_sense_keys":["principal%1:18:00::"],"candidates":[{"index":1,"sense_key":"principal%1:18:01::","synset_id":"star.n.04"},{"index":2,"sense_key":"principal%1:21:02::","synset_id":"principal.n.04"},{"index":3,"sense_key":"principal%1:18:00::","synset_id":"principal.n.02"},{"index":4,"sense_key":"principal%1:18:03::","synset_id":"principal.n.05"},{"index":5,"sense_key":"principal%1:18:02::","synset_id":"principal.n.06"},{"index":6,"sense_key":"principal%1:21:01::","synset_id":"principal.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"principal%1:18:00::","call_ids":["senseval2.d002.s076.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"principal%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00485,"input_uncached_usd":0.0029300000000000003,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6654394579818472}
{"item_id":"senseval2.d002.s076.t005","gold_sense_keys":["measure%1:03:00::"],"candidates":[{"index":1,"sense_key":"measure%1:10:02::","synset_id":"bill.n.01"},{"index":2,"sense_key":"measure%1:03:00::","synset_id":"measure.n.02"},{"index":3,"sense_key":"measure%1:06:00::","synset_id":"measuring_stick.n.01"},{"index":4,"sense_key":"measure%1:06:01::","synset_id":"measure.n.09"},{"index":5,"sense_key":"measure%1:04:00::","synset_id":"measurement.n.01"},{"index":6,"sense_key":"measure%1:10:00::","synset_id":"measure.n.07"},{"index":7,"sense_key":"measure%1:10:03::","synset_id":"standard.n.01"},{"index":8,"sense_key":"measure%1:10:01::","synset_id":"meter.n.03"},{"index":9,"sense_key":"measure%1:04:01::","synset_id":"measure.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"measure%1:03:00::","call_ids":["senseval2.d002.s076.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"measure%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.005845,"input_uncached_usd":0.0038350000000000003,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6286116250557825}
{"item_id":"senseval2.d002.s076.t006","gold_sense_keys":["authority%1:07:00::"],"candidates":[{"index":1,"sense_key":"authority%1:09:00::","synset_id":"assurance.n.01"},{"index":2,"sense_key":"authority%1:07:00::","synset_id":"authority.n.01"},{"index":3,"sense_key":"authority%1:07:02::","synset_id":"authority.n.06"},{"index":4,"sense_key":"authority%1:14:00::","synset_id":"agency.n.01"},{"index":5,"sense_key":"authority%1:18:00::","synset_id":"authority.n.03"},{"index":6,"sense_key":"authority%1:10:00::","synset_id":"authority.n.07"},{"index":7,"sense_key":"authority%1:18:01::","synset_id":"authority.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"authority%1:07:00::","call_ids":["senseval2.d002.s076.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"authority%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":690,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.006,"input_uncached_usd":0.0034500000000000004,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.735204334021546}
{"item_id":"senseval2.d002.s076.t007","gold_sense_keys":["faculty%1:14:00::"],"candidates":[{"index":1,"sense_key":"faculty%1:14:00::","synset_id":"staff.n.03"},{"index":2,"sense_key":"faculty%1:09:00::","synset_id":"faculty.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"faculty%1:14:00::","call_ids":["senseval2.d002.s076.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"faculty%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00404,"input_uncached_usd":0.0022400000000000002,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.814264832995832}
{"item_id":"senseval2.d002.s076.t009","gold_sense_keys":["matter%1:09:01::"],"candidates":[{"index":1,"sense_key":"matter%1:09:02::","synset_id":"matter.n.04"},{"index":2,"sense_key":"matter%1:07:00::","synset_id":"matter.n.05"},{"index":3,"sense_key":"matter%1:10:00::","synset_id":"matter.n.06"},{"index":4,"sense_key":"matter%1:03:00::","synset_id":"matter.n.03"},{"index":5,"sense_key":"matter%1:09:00::","synset_id":"topic.n.02"},{"index":6,"sense_key":"matter%1:09:01::","synset_id":"matter.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"matter%1:09:01::","call_ids":["senseval2.d002.s076.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"matter%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":174,"reasoning_output_tokens":158},"cost":{"currency":"USD","total_usd":0.008270000000000001,"input_uncached_usd":0.00305,"input_cached_usd":0.0,"output_usd":0.00522,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.805811292026192}
{"item_id":"senseval2.d002.s076.t010","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"},{"index":2,"sense_key":"student%1:18:00::","synset_id":"student.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s076.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004225,"input_uncached_usd":0.002245,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9685880420729518}
{"item_id":"senseval2.d002.s077.t000","gold_sense_keys":["study%1:10:00::"],"candidates":[{"index":1,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":2,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":3,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":4,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":5,"sense_key":"study%1:10:01::","synset_id":"study.n.10"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":8,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":9,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":10,"sense_key":"study%1:06:00::","synset_id":"study.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"study%1:10:00::","call_ids":["senseval2.d002.s077.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":198,"reasoning_output_tokens":182},"cost":{"currency":"USD","total_usd":0.009795000000000002,"input_uncached_usd":0.0038550000000000004,"input_cached_usd":0.0,"output_usd":0.00594,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.33836779196281}
{"item_id":"senseval2.d002.s077.t001","gold_sense_keys":["study%1:10:00::"],"candidates":[{"index":1,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":2,"sense_key":"study%1:10:01::","synset_id":"study.n.10"},{"index":3,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":4,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":5,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":6,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":7,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":8,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":9,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":10,"sense_key":"study%1:09:04::","synset_id":"study.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"study%1:10:00::","call_ids":["senseval2.d002.s077.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":141},"cost":{"currency":"USD","total_usd":0.008565000000000001,"input_uncached_usd":0.0038550000000000004,"input_cached_usd":0.0,"output_usd":0.00471,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.045035207993351}
{"item_id":"senseval2.d002.s077.t002","gold_sense_keys":["most%4:02:00::"],"candidates":[{"index":1,"sense_key":"most%4:02:02::","synset_id":"about.r.07"},{"index":2,"sense_key":"most%4:02:00::","synset_id":"most.r.01"},{"index":3,"sense_key":"most%4:02:01::","synset_id":"most.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"most%4:02:00::","call_ids":["senseval2.d002.s077.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"most%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.004435,"input_uncached_usd":0.002455,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8978207079926506}
{"item_id":"senseval2.d002.s077.t004","gold_sense_keys":["tell%2:32:00::"],"candidates":[{"index":1,"sense_key":"tell%2:31:01::","synset_id":"distinguish.v.01"},{"index":2,"sense_key":"tell%2:32:00::","synset_id":"tell.v.02"},{"index":3,"sense_key":"tell%2:32:04::","synset_id":"state.v.01"},{"index":4,"sense_key":"tell%2:32:01::","synset_id":"tell.v.03"},{"index":5,"sense_key":"tell%2:32:08::","synset_id":"assure.v.02"},{"index":6,"sense_key":"tell%2:32:03::","synset_id":"tell.v.07"},{"index":7,"sense_key":"tell%2:31:00::","synset_id":"tell.v.05"},{"index":8,"sense_key":"tell%2:32:02::","synset_id":"order.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tell%2:32:00::","call_ids":["senseval2.d002.s077.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tell%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":1564,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.05038,"input_uncached_usd":0.0034600000000000004,"input_cached_usd":0.0,"output_usd":0.04692,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":24.196441833046265}
{"item_id":"senseval2.d002.s077.t005","gold_sense_keys":["best%3:00:00::"],"candidates":[{"index":1,"sense_key":"dear%5:00:00:close:02","synset_id":"dear.s.02"},{"index":2,"sense_key":"best%5:00:00:advisable:00","synset_id":"better.s.01"},{"index":3,"sense_key":"good%5:00:00:righteous:00","synset_id":"good.s.05"},{"index":4,"sense_key":"good%5:00:00:opportune:00","synset_id":"good.s.10"},{"index":5,"sense_key":"good%5:00:00:complete:00","synset_id":"good.s.07"},{"index":6,"sense_key":"good%5:00:00:fresh:01","synset_id":"good.s.18"},{"index":7,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":8,"sense_key":"good%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":9,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":10,"sense_key":"good%5:00:00:nice:00","synset_id":"good.s.04"},{"index":11,"sense_key":"good%3:00:02::","synset_id":"good.a.03"},{"index":12,"sense_key":"beneficial%5:00:00:advantageous:00","synset_id":"beneficial.s.01"},{"index":13,"sense_key":"good%3:00:01::","synset_id":"good.a.01"},{"index":14,"sense_key":"good%5:00:00:pleasing:00","synset_id":"good.s.13"},{"index":15,"sense_key":"best%3:00:00::","synset_id":"best.a.01"},{"index":16,"sense_key":"adept%5:00:00:skilled:00","synset_id":"adept.s.01"},{"index":17,"sense_key":"good%5:00:00:healthful:00","synset_id":"good.s.16"},{"index":18,"sense_key":"estimable%5:00:00:reputable:00","synset_id":"estimable.s.01"},{"index":19,"sense_key":"dependable%5:00:00:sound:00","synset_id":"dependable.s.03"},{"index":20,"sense_key":"good%5:00:00:healthy:00","synset_id":"good.s.15"},{"index":21,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":22,"sense_key":"good%5:00:00:discriminating:00","synset_id":"good.s.19"},{"index":23,"sense_key":"good%5:00:00:genuine:00","synset_id":"good.s.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"best%3:00:00::","call_ids":["senseval2.d002.s077.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"best%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1396,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":97},"cost":{"currency":"USD","total_usd":0.01037,"input_uncached_usd":0.006980000000000001,"input_cached_usd":0.0,"output_usd":0.0033900000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2669291249476373}
{"item_id":"senseval2.d002.s077.t006","gold_sense_keys":["school%1:14:00::"],"candidates":[{"index":1,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":2,"sense_key":"school%1:14:01::","synset_id":"school.n.07"},{"index":3,"sense_key":"school%1:28:00::","synset_id":"school.n.05"},{"index":4,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":5,"sense_key":"school%1:09:00::","synset_id":"school.n.03"},{"index":6,"sense_key":"school%1:06:00::","synset_id":"school.n.02"},{"index":7,"sense_key":"school%1:14:00::","synset_id":"school.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"school%1:14:00::","call_ids":["senseval2.d002.s077.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"school%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":106},"cost":{"currency":"USD","total_usd":0.006895,"input_uncached_usd":0.003265,"input_cached_usd":0.0,"output_usd":0.00363,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7691583749838173}
{"item_id":"senseval2.d002.s077.t007","gold_sense_keys":["outside%5:00:00:external:00","outside%5:00:00:extrinsic:00"],"candidates":[{"index":1,"sense_key":"outside%5:00:00:exterior:00","synset_id":"outside.s.04"},{"index":2,"sense_key":"outside%5:00:00:external:00","synset_id":"outside.s.02"},{"index":3,"sense_key":"outside%5:00:00:extrinsic:00","synset_id":"external.s.01"},{"index":4,"sense_key":"outside%5:00:00:outer:00","synset_id":"outside.s.07"},{"index":5,"sense_key":"outside%5:00:00:foreign:02","synset_id":"external.s.02"},{"index":6,"sense_key":"outside%3:00:00::","synset_id":"outside.a.01"},{"index":7,"sense_key":"outside%5:00:00:inaccurate:00","synset_id":"away.s.02"},{"index":8,"sense_key":"outside%5:00:00:unlikely:00","synset_id":"outside.s.06"},{"index":9,"sense_key":"outside%3:00:04::","synset_id":"outdoor.a.01"},{"index":10,"sense_key":"outside%5:00:00:extramural:00","synset_id":"outside.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"outside%5:00:00:external:00","call_ids":["senseval2.d002.s077.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"outside%5:00:00:external:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":778,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01973,"input_uncached_usd":0.0038900000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.510855250060558}
{"item_id":"senseval2.d002.s077.t008","gold_sense_keys":["interference%1:04:00::"],"candidates":[{"index":1,"sense_key":"interference%1:04:00::","synset_id":"hindrance.n.03"},{"index":2,"sense_key":"interference%1:11:00::","synset_id":"noise.n.03"},{"index":3,"sense_key":"interference%1:06:00::","synset_id":"hindrance.n.02"},{"index":4,"sense_key":"interference%1:04:01::","synset_id":"interference.n.04"},{"index":5,"sense_key":"interference%1:10:00::","synset_id":"intervention.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"interference%1:04:00::","call_ids":["senseval2.d002.s077.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"interference%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":168,"reasoning_output_tokens":153},"cost":{"currency":"USD","total_usd":0.007755000000000001,"input_uncached_usd":0.0027150000000000004,"input_cached_usd":0.0,"output_usd":0.00504,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2475231669377536}
{"item_id":"senseval2.d002.s077.t009","gold_sense_keys":["govern%2:41:00::"],"candidates":[{"index":1,"sense_key":"govern%2:41:01::","synset_id":"govern.v.02"},{"index":2,"sense_key":"govern%2:42:00::","synset_id":"govern.v.04"},{"index":3,"sense_key":"govern%2:41:00::","synset_id":"govern.v.03"},{"index":4,"sense_key":"govern%2:41:02::","synset_id":"regulate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"govern%2:41:00::","call_ids":["senseval2.d002.s077.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"govern%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.005585,"input_uncached_usd":0.0026750000000000003,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0787747090216726}
{"item_id":"senseval2.d002.s077.t010","gold_sense_keys":["powerful%3:00:00::","powerful%5:00:00:influential:00"],"candidates":[{"index":1,"sense_key":"powerful%5:00:01:strong:00","synset_id":"brawny.s.01"},{"index":2,"sense_key":"powerful%5:00:00:superhuman:00","synset_id":"herculean.s.01"},{"index":3,"sense_key":"powerful%5:00:02:strong:00","synset_id":"knock-down.s.01"},{"index":4,"sense_key":"powerful%5:00:00:influential:00","synset_id":"potent.s.01"},{"index":5,"sense_key":"powerful%3:00:00::","synset_id":"powerful.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"powerful%3:00:00::","call_ids":["senseval2.d002.s077.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"powerful%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01868,"input_uncached_usd":0.00284,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.030342707992531}
{"item_id":"senseval2.d002.s077.t011","gold_sense_keys":["head%1:18:01::"],"candidates":[{"index":1,"sense_key":"head%1:11:01::","synset_id":"pass.n.09"},{"index":2,"sense_key":"head%1:09:00::","synset_id":"mind.n.01"},{"index":3,"sense_key":"head%1:18:00::","synset_id":"head.n.04"},{"index":4,"sense_key":"head%1:15:01::","synset_id":"fountainhead.n.02"},{"index":5,"sense_key":"head%1:17:02::","synset_id":"promontory.n.01"},{"index":6,"sense_key":"head%1:06:00::","synset_id":"head.n.31"},{"index":7,"sense_key":"head%1:08:00::","synset_id":"head.n.01"},{"index":8,"sense_key":"head%1:10:02::","synset_id":"question.n.02"},{"index":9,"sense_key":"head%1:05:00::","synset_id":"head.n.02"},{"index":10,"sense_key":"head%1:15:02::","synset_id":"forefront.n.01"},{"index":11,"sense_key":"head%1:06:02::","synset_id":"head.n.30"},{"index":12,"sense_key":"head%1:14:00::","synset_id":"head.n.05"},{"index":13,"sense_key":"head%1:10:00::","synset_id":"heading.n.01"},{"index":14,"sense_key":"head%1:18:03::","synset_id":"head.n.14"},{"index":15,"sense_key":"head%1:18:02::","synset_id":"head.n.15"},{"index":16,"sense_key":"head%1:08:02::","synset_id":"head.n.26"},{"index":17,"sense_key":"head%1:19:00::","synset_id":"head.n.06"},{"index":18,"sense_key":"head%1:10:03::","synset_id":"head.n.09"},{"index":19,"sense_key":"head%1:15:00::","synset_id":"head.n.07"},{"index":20,"sense_key":"head%1:06:05::","synset_id":"read/write_head.n.01"},{"index":21,"sense_key":"head%1:23:00::","synset_id":"head.n.11"},{"index":22,"sense_key":"head%1:20:00::","synset_id":"capitulum.n.01"},{"index":23,"sense_key":"head%1:17:01::","synset_id":"head.n.17"},{"index":24,"sense_key":"head%1:11:00::","synset_id":"headway.n.02"},{"index":25,"sense_key":"head%1:17:00::","synset_id":"head.n.18"},{"index":26,"sense_key":"head%1:04:00::","synset_id":"oral_sex.n.01"},{"index":27,"sense_key":"head%1:26:00::","synset_id":"head.n.10"},{"index":28,"sense_key":"head%1:08:01::","synset_id":"head.n.25"},{"index":29,"sense_key":"head%1:06:03::","synset_id":"head.n.29"},{"index":30,"sense_key":"head%1:06:01::","synset_id":"drumhead.n.01"},{"index":31,"sense_key":"head%1:10:01::","synset_id":"point.n.20"},{"index":32,"sense_key":"head%1:06:04::","synset_id":"head.n.28"},{"index":33,"sense_key":"head%1:18:01::","synset_id":"principal.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":33,"chosen_sense_key":"head%1:18:01::","call_ids":["senseval2.d002.s077.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":33,"predicted_sense_key":"head%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1640,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":129},"cost":{"currency":"USD","total_usd":0.012550000000000002,"input_uncached_usd":0.0082,"input_cached_usd":0.0,"output_usd":0.00435,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.873475542059168}
{"item_id":"senseval2.d002.s078.t000","gold_sense_keys":["authority%1:07:00::"],"candidates":[{"index":1,"sense_key":"authority%1:07:00::","synset_id":"authority.n.01"},{"index":2,"sense_key":"authority%1:07:02::","synset_id":"authority.n.06"},{"index":3,"sense_key":"authority%1:18:00::","synset_id":"authority.n.03"},{"index":4,"sense_key":"authority%1:10:00::","synset_id":"authority.n.07"},{"index":5,"sense_key":"authority%1:09:00::","synset_id":"assurance.n.01"},{"index":6,"sense_key":"authority%1:14:00::","synset_id":"agency.n.01"},{"index":7,"sense_key":"authority%1:18:01::","synset_id":"authority.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"authority%1:07:00::","call_ids":["senseval2.d002.s078.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"authority%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":676,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":79},"cost":{"currency":"USD","total_usd":0.006200000000000001,"input_uncached_usd":0.00338,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6314179580658674}
{"item_id":"senseval2.d002.s078.t002","gold_sense_keys":["go_with%2:42:01::"],"candidates":[{"index":1,"sense_key":"go_with%2:42:01::","synset_id":"attach_to.v.01"},{"index":2,"sense_key":"go_with%2:42:00::","synset_id":"collocate_with.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"go_with%2:42:01::","call_ids":["senseval2.d002.s078.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"go_with%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":110},"cost":{"currency":"USD","total_usd":0.0060750000000000005,"input_uncached_usd":0.0023250000000000002,"input_cached_usd":0.0,"output_usd":0.0037500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.592765082954429}
{"item_id":"senseval2.d002.s078.t003","gold_sense_keys":["unambiguous%3:00:04::"],"candidates":[{"index":1,"sense_key":"unambiguous%3:00:00::","synset_id":"unambiguous.a.01"},{"index":2,"sense_key":"unambiguous%3:00:04::","synset_id":"unequivocal.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"unambiguous%3:00:04::","call_ids":["senseval2.d002.s078.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"unambiguous%3:00:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":468,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.00474,"input_uncached_usd":0.00234,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3163300829473883}
{"item_id":"senseval2.d002.s079.t000","gold_sense_keys":["school%1:14:00::"],"candidates":[{"index":1,"sense_key":"school%1:09:00::","synset_id":"school.n.03"},{"index":2,"sense_key":"school%1:06:00::","synset_id":"school.n.02"},{"index":3,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":4,"sense_key":"school%1:14:01::","synset_id":"school.n.07"},{"index":5,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":6,"sense_key":"school%1:14:00::","synset_id":"school.n.01"},{"index":7,"sense_key":"school%1:28:00::","synset_id":"school.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"school%1:14:00::","call_ids":["senseval2.d002.s079.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"school%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":134},"cost":{"currency":"USD","total_usd":0.00784,"input_uncached_usd":0.00334,"input_cached_usd":0.0,"output_usd":0.0045000000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1288661250146106}
{"item_id":"senseval2.d002.s079.t002","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":2,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":5,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":6,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":7,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":8,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":9,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":10,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":11,"sense_key":"way%1:21:00::","synset_id":"way.n.12"},{"index":12,"sense_key":"way%1:09:00::","synset_id":"way.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"way%1:07:01::","call_ids":["senseval2.d002.s079.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":843,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.006225,"input_uncached_usd":0.004215,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.246671541943215}
{"item_id":"senseval2.d002.s079.t003","gold_sense_keys":["produce%2:36:03::"],"candidates":[{"index":1,"sense_key":"produce%2:39:00::","synset_id":"produce.v.06"},{"index":2,"sense_key":"produce%2:36:05::","synset_id":"grow.v.07"},{"index":3,"sense_key":"produce%2:36:00::","synset_id":"produce.v.02"},{"index":4,"sense_key":"produce%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"produce%2:36:02::","synset_id":"produce.v.01"},{"index":6,"sense_key":"produce%2:36:03::","synset_id":"produce.v.03"},{"index":7,"sense_key":"produce%2:39:01::","synset_id":"produce.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"produce%2:36:02::","call_ids":["senseval2.d002.s079.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"produce%2:36:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":712,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.03494,"input_uncached_usd":0.0035600000000000002,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":21.446738000027835}
{"item_id":"senseval2.d002.s079.t004","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"},{"index":2,"sense_key":"student%1:18:00::","synset_id":"student.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s079.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.0038250000000000003,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.362700958037749}
{"item_id":"senseval2.d002.s079.t005","gold_sense_keys":["high%3:00:02::"],"candidates":[{"index":1,"sense_key":"high%3:00:01::","synset_id":"high.a.02"},{"index":2,"sense_key":"high%5:00:00:malodorous:00","synset_id":"gamey.s.02"},{"index":3,"sense_key":"high%3:00:03::","synset_id":"high.a.04"},{"index":4,"sense_key":"high%5:00:00:intoxicated:00","synset_id":"high.s.04"},{"index":5,"sense_key":"high%5:00:00:elated:00","synset_id":"high.s.02"},{"index":6,"sense_key":"high%3:00:02::","synset_id":"high.a.01"},{"index":7,"sense_key":"high%5:00:00:superior:01","synset_id":"eminent.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"high%3:00:02::","call_ids":["senseval2.d002.s079.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"high%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":650,"cached_input_tokens":0,"output_tokens":518,"reasoning_output_tokens":502},"cost":{"currency":"USD","total_usd":0.01879,"input_uncached_usd":0.0032500000000000003,"input_cached_usd":0.0,"output_usd":0.01554,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.697461375035346}
{"item_id":"senseval2.d002.s079.t006","gold_sense_keys":["morale%1:26:00::"],"candidates":[{"index":1,"sense_key":"morale%1:07:00::","synset_id":"esprit_de_corps.n.01"},{"index":2,"sense_key":"morale%1:26:00::","synset_id":"morale.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"morale%1:26:00::","call_ids":["senseval2.d002.s079.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"morale%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":191,"reasoning_output_tokens":175},"cost":{"currency":"USD","total_usd":0.007975000000000001,"input_uncached_usd":0.002245,"input_cached_usd":0.0,"output_usd":0.00573,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.562962790951133}
{"item_id":"senseval2.d002.s079.t007","gold_sense_keys":["superior%3:00:02::"],"candidates":[{"index":1,"sense_key":"superior%5:00:00:unaffected:00","synset_id":"superior.s.03"},{"index":2,"sense_key":"superior%3:00:01::","synset_id":"superior.a.02"},{"index":3,"sense_key":"superior%3:00:00::","synset_id":"superscript.a.01"},{"index":4,"sense_key":"superior%3:00:03::","synset_id":"superior.a.05"},{"index":5,"sense_key":"superior%5:00:00:senior:00","synset_id":"ranking.s.01"},{"index":6,"sense_key":"superior%5:00:00:dominant:01","synset_id":"superior.s.01"},{"index":7,"sense_key":"superior%3:00:02::","synset_id":"superior.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"superior%3:00:02::","call_ids":["senseval2.d002.s079.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"superior%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.0054150000000000005,"input_uncached_usd":0.0033450000000000003,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0931931249797344}
{"item_id":"senseval2.d002.s079.t008","gold_sense_keys":["academic%3:01:00::"],"candidates":[{"index":1,"sense_key":"academic%3:01:00::","synset_id":"academic.a.01"},{"index":2,"sense_key":"academic%5:00:00:theoretical:01","synset_id":"academic.s.01"},{"index":3,"sense_key":"academic%5:00:00:scholarly:00","synset_id":"academic.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"academic%3:01:00::","call_ids":["senseval2.d002.s079.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"academic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004450000000000001,"input_uncached_usd":0.0024700000000000004,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4587754159001634}
{"item_id":"senseval2.d002.s079.t009","gold_sense_keys":["performance%1:04:00::","performance%1:04:03::"],"candidates":[{"index":1,"sense_key":"performance%1:04:01::","synset_id":"performance.n.02"},{"index":2,"sense_key":"performance%1:04:03::","synset_id":"performance.n.04"},{"index":3,"sense_key":"performance%1:22:00::","synset_id":"operation.n.08"},{"index":4,"sense_key":"performance%1:10:00::","synset_id":"performance.n.01"},{"index":5,"sense_key":"performance%1:04:00::","synset_id":"performance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"performance%1:04:00::","call_ids":["senseval2.d002.s079.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"performance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":444,"reasoning_output_tokens":428},"cost":{"currency":"USD","total_usd":0.01643,"input_uncached_usd":0.0031100000000000004,"input_cached_usd":0.0,"output_usd":0.01332,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.03645537502598}
{"item_id":"senseval2.d002.s080.t000","gold_sense_keys":["fact%1:10:01::"],"candidates":[{"index":1,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":2,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"},{"index":3,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":4,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fact%1:10:01::","call_ids":["senseval2.d002.s080.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fact%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01848,"input_uncached_usd":0.0026400000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.619704334065318}
{"item_id":"senseval2.d002.s080.t001","gold_sense_keys":["ruffle%2:37:01::"],"candidates":[{"index":1,"sense_key":"ruffle%2:35:00::","synset_id":"ruffle.v.09"},{"index":2,"sense_key":"ruffle%2:38:01::","synset_id":"tittup.v.01"},{"index":3,"sense_key":"ruffle%2:35:02::","synset_id":"ruffle.v.08"},{"index":4,"sense_key":"ruffle%2:35:03::","synset_id":"flick.v.06"},{"index":5,"sense_key":"ruffle%2:38:00::","synset_id":"ripple.v.01"},{"index":6,"sense_key":"ruffle%2:35:01::","synset_id":"ruffle.v.07"},{"index":7,"sense_key":"ruffle%2:35:04::","synset_id":"shuffle.v.03"},{"index":8,"sense_key":"ruffle%2:37:01::","synset_id":"ruffle.v.04"},{"index":9,"sense_key":"ruffle%2:37:00::","synset_id":"ruffle.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"ruffle%2:37:01::","call_ids":["senseval2.d002.s080.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"ruffle%2:37:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":736,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":112},"cost":{"currency":"USD","total_usd":0.007520000000000001,"input_uncached_usd":0.00368,"input_cached_usd":0.0,"output_usd":0.00384,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.945781624992378}
{"item_id":"senseval2.d002.s080.t002","gold_sense_keys":["fact%1:10:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":2,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":3,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"},{"index":4,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"fact%1:10:01::","call_ids":["senseval2.d002.s080.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"fact%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018475,"input_uncached_usd":0.002635,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.179187332978472}
{"item_id":"senseval2.d002.s080.t004","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval2.d002.s080.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":118},"cost":{"currency":"USD","total_usd":0.006285000000000001,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.00402,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1407682090066373}
{"item_id":"senseval2.d002.s080.t005","gold_sense_keys":["so%4:02:02::"],"candidates":[{"index":1,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":2,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":3,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":4,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":5,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":6,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":7,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":8,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":9,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":10,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"so%4:02:02::","call_ids":["senseval2.d002.s080.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"so%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":837,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020025,"input_uncached_usd":0.004185,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.98371008306276}
{"item_id":"senseval2.d002.s081.t000","gold_sense_keys":["publish%2:32:00::"],"candidates":[{"index":1,"sense_key":"publish%2:32:00::","synset_id":"publish.v.02"},{"index":2,"sense_key":"publish%2:36:01::","synset_id":"print.v.01"},{"index":3,"sense_key":"publish%2:36:00::","synset_id":"publish.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"publish%2:32:00::","call_ids":["senseval2.d002.s081.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"publish%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":138},"cost":{"currency":"USD","total_usd":0.006815,"input_uncached_usd":0.002225,"input_cached_usd":0.0,"output_usd":0.00459,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3450952499406412}
{"item_id":"senseval3.d000.s000.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":3,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":4,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":5,"sense_key":"man%1:05:00::","synset_id":"world.n.08"},{"index":6,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":7,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":8,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":9,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":10,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":11,"sense_key":"man%1:15:00::","synset_id":"man.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.00604,"input_uncached_usd":0.00355,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9759057910414413}
{"item_id":"senseval3.d000.s001.t003","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":2,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":3,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":4,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":386,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00385,"input_uncached_usd":0.00193,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9664381670299917}
{"item_id":"senseval3.d000.s002.t000","gold_sense_keys":["have%2:40:00::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":3,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":4,"sense_key":"have%2:35:00::","synset_id":"take.v.35"},{"index":5,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":6,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":7,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":8,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":9,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":10,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":11,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":12,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":13,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":14,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":15,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":16,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":17,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":18,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":19,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d000.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1035,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":121},"cost":{"currency":"USD","total_usd":0.009255000000000001,"input_uncached_usd":0.005175000000000001,"input_cached_usd":0.0,"output_usd":0.00408,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5357547500170767}
{"item_id":"senseval3.d000.s002.t002","gold_sense_keys":["answer%1:10:01::"],"candidates":[{"index":1,"sense_key":"answer%1:10:00::","synset_id":"answer.n.03"},{"index":2,"sense_key":"answer%1:10:03::","synset_id":"answer.n.04"},{"index":3,"sense_key":"answer%1:10:01::","synset_id":"answer.n.01"},{"index":4,"sense_key":"answer%1:04:00::","synset_id":"answer.n.05"},{"index":5,"sense_key":"answer%1:10:02::","synset_id":"solution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"answer%1:10:01::","call_ids":["senseval3.d000.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"answer%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":135},"cost":{"currency":"USD","total_usd":0.00688,"input_uncached_usd":0.00238,"input_cached_usd":0.0,"output_usd":0.0045000000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.133007041993551}
{"item_id":"senseval3.d000.s002.t004","gold_sense_keys":["surprise%1:12:00::"],"candidates":[{"index":1,"sense_key":"surprise%1:11:00::","synset_id":"surprise.n.02"},{"index":2,"sense_key":"surprise%1:04:00::","synset_id":"surprise.n.03"},{"index":3,"sense_key":"surprise%1:12:00::","synset_id":"surprise.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"surprise%1:12:00::","call_ids":["senseval3.d000.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"surprise%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":280,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.0031100000000000004,"input_uncached_usd":0.0014000000000000002,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9250548749696463}
{"item_id":"senseval3.d000.s002.t005","gold_sense_keys":["fit%1:26:00::"],"candidates":[{"index":1,"sense_key":"fit%1:04:00::","synset_id":"burst.n.03"},{"index":2,"sense_key":"fit%1:26:01::","synset_id":"fit.n.01"},{"index":3,"sense_key":"fit%1:26:00::","synset_id":"paroxysm.n.01"},{"index":4,"sense_key":"fit%1:07:00::","synset_id":"fit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fit%1:26:00::","call_ids":["senseval3.d000.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fit%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":368,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.00334,"input_uncached_usd":0.00184,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8892713750246912}
{"item_id":"senseval3.d000.s003.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":2,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":3,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":4,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":5,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":6,"sense_key":"man%1:05:00::","synset_id":"world.n.08"},{"index":7,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":8,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":9,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":10,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":11,"sense_key":"man%1:18:08::","synset_id":"man.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":760,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.00593,"input_uncached_usd":0.0038000000000000004,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.882793791941367}
{"item_id":"senseval3.d000.s003.t001","gold_sense_keys":["drunk%3:00:00::"],"candidates":[{"index":1,"sense_key":"drunk%5:00:00:excited:00","synset_id":"intoxicated.s.01"},{"index":2,"sense_key":"drunk%3:00:00::","synset_id":"intoxicated.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"drunk%3:00:00::","call_ids":["senseval3.d000.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"drunk%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":329,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0035650000000000005,"input_uncached_usd":0.0016450000000000002,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5132869170047343}
{"item_id":"senseval3.d000.s003.t002","gold_sense_keys":["crazy%5:00:00:insane:00"],"candidates":[{"index":1,"sense_key":"crazy%5:00:00:impractical:00","synset_id":"crazy.s.02"},{"index":2,"sense_key":"crazy%5:00:00:insane:00","synset_id":"brainsick.s.01"},{"index":3,"sense_key":"crazy%5:00:00:enthusiastic:00","synset_id":"crazy.s.05"},{"index":4,"sense_key":"crazy%5:00:00:excited:00","synset_id":"crazy.s.03"},{"index":5,"sense_key":"crazy%5:00:00:strange:00","synset_id":"crazy.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"crazy%5:00:00:insane:00","call_ids":["senseval3.d000.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"crazy%5:00:00:insane:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.004710000000000001,"input_uncached_usd":0.00231,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.9377435829956084}
{"item_id":"senseval3.d000.s004.t002","gold_sense_keys":["match%2:42:01::"],"candidates":[{"index":1,"sense_key":"match%2:35:10::","synset_id":"match.v.07"},{"index":2,"sense_key":"match%2:40:00::","synset_id":"match.v.02"},{"index":3,"sense_key":"match%2:42:01::","synset_id":"equal.v.02"},{"index":4,"sense_key":"match%2:33:00::","synset_id":"pit.v.01"},{"index":5,"sense_key":"match%2:30:00::","synset_id":"match.v.09"},{"index":6,"sense_key":"match%2:41:00::","synset_id":"meet.v.06"},{"index":7,"sense_key":"match%2:30:07::","synset_id":"equal.v.03"},{"index":8,"sense_key":"match%2:30:01::","synset_id":"match.v.05"},{"index":9,"sense_key":"match%2:35:08::","synset_id":"match.v.03"},{"index":10,"sense_key":"match%2:42:00::","synset_id":"match.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"match%2:42:01::","call_ids":["senseval3.d000.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"match%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":689,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":193},"cost":{"currency":"USD","total_usd":0.009685,"input_uncached_usd":0.003445,"input_cached_usd":0.0,"output_usd":0.00624,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.176237209001556}
{"item_id":"senseval3.d000.s004.t003","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"},{"index":2,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":3,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":4,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":5,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":458,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":114},"cost":{"currency":"USD","total_usd":0.006190000000000001,"input_uncached_usd":0.0022900000000000004,"input_cached_usd":0.0,"output_usd":0.0039000000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.228602165938355}
{"item_id":"senseval3.d000.s004.t004","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"},{"index":2,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":3,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":4,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":5,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":458,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":116},"cost":{"currency":"USD","total_usd":0.00625,"input_uncached_usd":0.0022900000000000004,"input_cached_usd":0.0,"output_usd":0.00396,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.673675167025067}
{"item_id":"senseval3.d000.s004.t005","gold_sense_keys":["lose%2:40:02::"],"candidates":[{"index":1,"sense_key":"lose%2:30:03::","synset_id":"suffer.v.11"},{"index":2,"sense_key":"lose%2:40:06::","synset_id":"lose.v.07"},{"index":3,"sense_key":"lose%2:33:00::","synset_id":"lose.v.02"},{"index":4,"sense_key":"lose%2:33:01::","synset_id":"fall_back.v.04"},{"index":5,"sense_key":"lose%2:40:00::","synset_id":"lose.v.08"},{"index":6,"sense_key":"lose%2:35:00::","synset_id":"misplace.v.01"},{"index":7,"sense_key":"lose%2:40:01::","synset_id":"lose.v.05"},{"index":8,"sense_key":"lose%2:39:00::","synset_id":"lose.v.06"},{"index":9,"sense_key":"lose%2:39:01::","synset_id":"miss.v.01"},{"index":10,"sense_key":"lose%2:40:02::","synset_id":"lose.v.01"},{"index":11,"sense_key":"lose%2:37:00::","synset_id":"lose.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"lose%2:40:02::","call_ids":["senseval3.d000.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"lose%2:40:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":708,"cached_input_tokens":0,"output_tokens":204,"reasoning_output_tokens":188},"cost":{"currency":"USD","total_usd":0.00966,"input_uncached_usd":0.00354,"input_cached_usd":0.0,"output_usd":0.0061200000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.1420832079602405}
{"item_id":"senseval3.d000.s004.t006","gold_sense_keys":["count%1:23:00::"],"candidates":[{"index":1,"sense_key":"count%1:18:00::","synset_id":"count.n.03"},{"index":2,"sense_key":"count%1:23:00::","synset_id":"count.n.01"},{"index":3,"sense_key":"count%1:04:00::","synset_id":"count.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"count%1:23:00::","call_ids":["senseval3.d000.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"count%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":348,"cached_input_tokens":0,"output_tokens":406,"reasoning_output_tokens":390},"cost":{"currency":"USD","total_usd":0.01392,"input_uncached_usd":0.0017400000000000002,"input_cached_usd":0.0,"output_usd":0.01218,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.953616291983053}
{"item_id":"senseval3.d000.s004.t007","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":2,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":3,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":4,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":5,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":6,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":7,"sense_key":"man%1:05:00::","synset_id":"world.n.08"},{"index":8,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":9,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":10,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":11,"sense_key":"man%1:06:00::","synset_id":"man.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":781,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.005795000000000001,"input_uncached_usd":0.0039050000000000005,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8592432910809293}
{"item_id":"senseval3.d000.s004.t008","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"},{"index":2,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":3,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s004.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.00388,"input_uncached_usd":0.0022,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0433162089902908}
{"item_id":"senseval3.d000.s005.t000","gold_sense_keys":["guy%1:18:00::"],"candidates":[{"index":1,"sense_key":"guy%1:18:00::","synset_id":"guy.n.01"},{"index":2,"sense_key":"guy%1:06:01::","synset_id":"guy.n.02"},{"index":3,"sense_key":"guy%1:06:00::","synset_id":"guy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"guy%1:18:00::","call_ids":["senseval3.d000.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"guy%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":398,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.00394,"input_uncached_usd":0.00199,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0591842079302296}
{"item_id":"senseval3.d000.s005.t002","gold_sense_keys":["think%2:31:01::","think%2:31:03::","think%2:31:12::"],"candidates":[{"index":1,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":2,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":3,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":4,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":5,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":6,"sense_key":"think%2:30:00::","synset_id":"think.v.13"},{"index":7,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":8,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":9,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":10,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":11,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":12,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":13,"sense_key":"think%2:31:08::","synset_id":"think.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"think%2:31:01::","call_ids":["senseval3.d000.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"think%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":833,"cached_input_tokens":0,"output_tokens":520,"reasoning_output_tokens":504},"cost":{"currency":"USD","total_usd":0.019764999999999998,"input_uncached_usd":0.004165,"input_cached_usd":0.0,"output_usd":0.015600000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.76887837506365}
{"item_id":"senseval3.d000.s005.t004","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"},{"index":2,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":5,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.004225,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.418503124965355}
{"item_id":"senseval3.d000.s006.t000","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":2,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"},{"index":3,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":4,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":5,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.00387,"input_uncached_usd":0.00231,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.667495166999288}
{"item_id":"senseval3.d000.s006.t002","gold_sense_keys":["bright%3:00:00::"],"candidates":[{"index":1,"sense_key":"bright%5:00:00:colorful:00","synset_id":"bright.s.01"},{"index":2,"sense_key":"bright%5:00:00:auspicious:00","synset_id":"bright.s.08"},{"index":3,"sense_key":"bright%5:00:00:polished:00","synset_id":"bright.s.04"},{"index":4,"sense_key":"bright%3:00:00::","synset_id":"bright.a.01"},{"index":5,"sense_key":"bright%5:00:00:reverberant:00","synset_id":"bright.s.06"},{"index":6,"sense_key":"bright%3:00:02::","synset_id":"undimmed.a.01"},{"index":7,"sense_key":"bright%5:00:00:intelligent:00","synset_id":"bright.s.02"},{"index":8,"sense_key":"bright%5:00:00:light:06","synset_id":"bright.s.03"},{"index":9,"sense_key":"bright%5:00:00:glorious:00","synset_id":"bright.s.05"},{"index":10,"sense_key":"bright%5:00:00:happy:00","synset_id":"bright.s.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"bright%3:00:00::","call_ids":["senseval3.d000.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"bright%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":744,"cached_input_tokens":0,"output_tokens":164,"reasoning_output_tokens":148},"cost":{"currency":"USD","total_usd":0.008640000000000002,"input_uncached_usd":0.00372,"input_cached_usd":0.0,"output_usd":0.00492,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.669589834054932}
{"item_id":"senseval3.d000.s006.t003","gold_sense_keys":["strange%3:00:00::"],"candidates":[{"index":1,"sense_key":"strange%3:00:00::","synset_id":"strange.a.01"},{"index":2,"sense_key":"strange%3:00:01::","synset_id":"foreign.a.02"},{"index":3,"sense_key":"strange%5:00:01:unfamiliar:00","synset_id":"strange.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"strange%3:00:00::","call_ids":["senseval3.d000.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"strange%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":411,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.005294999999999999,"input_uncached_usd":0.002055,"input_cached_usd":0.0,"output_usd":0.0032400000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.24785675003659}
{"item_id":"senseval3.d000.s006.t004","gold_sense_keys":["intensity%1:07:00::"],"candidates":[{"index":1,"sense_key":"intensity%1:07:02::","synset_id":"volume.n.06"},{"index":2,"sense_key":"intensity%1:07:03::","synset_id":"intensity.n.01"},{"index":3,"sense_key":"intensity%1:07:01::","synset_id":"saturation.n.04"},{"index":4,"sense_key":"intensity%1:07:00::","synset_id":"intensity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"intensity%1:07:00::","call_ids":["senseval3.d000.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"intensity%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":105},"cost":{"currency":"USD","total_usd":0.005825,"input_uncached_usd":0.0021950000000000003,"input_cached_usd":0.0,"output_usd":0.00363,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8422392499633133}
{"item_id":"senseval3.d000.s006.t005","gold_sense_keys":["cold%3:00:02::"],"candidates":[{"index":1,"sense_key":"cold%5:00:00:inhumane:00","synset_id":"cold.s.07"},{"index":2,"sense_key":"cold%5:00:00:dead:01","synset_id":"cold.s.11"},{"index":3,"sense_key":"cold%5:00:00:perfect:00","synset_id":"cold.s.03"},{"index":4,"sense_key":"cold%5:00:00:unenthusiastic:00","synset_id":"cold.s.08"},{"index":5,"sense_key":"cold%5:00:00:far:00","synset_id":"cold.s.10"},{"index":6,"sense_key":"cold%5:00:00:intense:00","synset_id":"cold.s.05"},{"index":7,"sense_key":"cold%5:00:00:cool:03","synset_id":"cold.s.02"},{"index":8,"sense_key":"cold%5:00:00:unloving:00","synset_id":"cold.s.06"},{"index":9,"sense_key":"cold%3:00:01::","synset_id":"cold.a.01"},{"index":10,"sense_key":"cold%5:00:00:unconscious:00","synset_id":"cold.s.09"},{"index":11,"sense_key":"cold%3:00:02::","synset_id":"cold.a.02"},{"index":12,"sense_key":"cold%5:00:00:stale:00","synset_id":"cold.s.01"},{"index":13,"sense_key":"cold%5:00:00:unoriginal:00","synset_id":"cold.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"cold%5:00:00:intense:00","call_ids":["senseval3.d000.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"cold%5:00:00:intense:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":808,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019880000000000002,"input_uncached_usd":0.00404,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.122884541982785}
{"item_id":"senseval3.d000.s006.t006","gold_sense_keys":["fire%1:12:00::"],"candidates":[{"index":1,"sense_key":"fire%1:11:00::","synset_id":"fire.n.01"},{"index":2,"sense_key":"fire%1:06:00::","synset_id":"fire.n.04"},{"index":3,"sense_key":"fire%1:10:00::","synset_id":"fire.n.09"},{"index":4,"sense_key":"fire%1:22:00::","synset_id":"fire.n.03"},{"index":5,"sense_key":"fire%1:11:01::","synset_id":"fire.n.08"},{"index":6,"sense_key":"fire%1:04:00::","synset_id":"fire.n.02"},{"index":7,"sense_key":"fire%1:12:00::","synset_id":"ardor.n.03"},{"index":8,"sense_key":"fire%1:27:01::","synset_id":"fire.n.05"},{"index":9,"sense_key":"fire%1:27:00::","synset_id":"fire.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"fire%1:12:00::","call_ids":["senseval3.d000.s006.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"fire%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":113},"cost":{"currency":"USD","total_usd":0.0071600000000000006,"input_uncached_usd":0.0032900000000000004,"input_cached_usd":0.0,"output_usd":0.00387,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.128314625006169}
{"item_id":"senseval3.d000.s006.t007","gold_sense_keys":["burn%2:43:01::"],"candidates":[{"index":1,"sense_key":"burn%2:30:02::","synset_id":"burn.v.01"},{"index":2,"sense_key":"burn%2:36:05::","synset_id":"cut.v.21"},{"index":3,"sense_key":"burn%2:43:01::","synset_id":"burn.v.02"},{"index":4,"sense_key":"burn%2:37:00::","synset_id":"burn.v.06"},{"index":5,"sense_key":"burn%2:41:00::","synset_id":"burn.v.08"},{"index":6,"sense_key":"burn%2:30:00::","synset_id":"burn.v.03"},{"index":7,"sense_key":"burn%2:39:04::","synset_id":"burn.v.10"},{"index":8,"sense_key":"burn%2:30:08::","synset_id":"cauterize.v.01"},{"index":9,"sense_key":"burn%2:34:00::","synset_id":"burn_off.v.01"},{"index":10,"sense_key":"burn%2:39:00::","synset_id":"bite.v.02"},{"index":11,"sense_key":"burn%2:30:01::","synset_id":"burn.v.07"},{"index":12,"sense_key":"burn%2:29:00::","synset_id":"sunburn.v.01"},{"index":13,"sense_key":"burn%2:40:01::","synset_id":"burn.v.09"},{"index":14,"sense_key":"burn%2:30:03::","synset_id":"burn.v.15"},{"index":15,"sense_key":"burn%2:43:02::","synset_id":"burn.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"burn%2:43:01::","call_ids":["senseval3.d000.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"burn%2:43:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":312,"reasoning_output_tokens":296},"cost":{"currency":"USD","total_usd":0.013555000000000001,"input_uncached_usd":0.004195,"input_cached_usd":0.0,"output_usd":0.00936,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.868784083984792}
{"item_id":"senseval3.d000.s007.t000","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d000.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":353,"reasoning_output_tokens":337},"cost":{"currency":"USD","total_usd":0.012870000000000001,"input_uncached_usd":0.0022800000000000003,"input_cached_usd":0.0,"output_usd":0.01059,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.936284665949643}
{"item_id":"senseval3.d000.s008.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":4,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":5,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":6,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":7,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":8,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":9,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":10,"sense_key":"man%1:05:00::","synset_id":"world.n.08"},{"index":11,"sense_key":"man%1:18:06::","synset_id":"man.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":785,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.006475000000000001,"input_uncached_usd":0.0039250000000000005,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.8184800000162795}
{"item_id":"senseval3.d000.s008.t001","gold_sense_keys":["drunk%3:00:00::"],"candidates":[{"index":1,"sense_key":"drunk%5:00:00:excited:00","synset_id":"intoxicated.s.01"},{"index":2,"sense_key":"drunk%3:00:00::","synset_id":"intoxicated.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"drunk%3:00:00::","call_ids":["senseval3.d000.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"drunk%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":354,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.00474,"input_uncached_usd":0.00177,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.58792750001885}
{"item_id":"senseval3.d000.s009.t000","gold_sense_keys":["wonder%2:31:00::"],"candidates":[{"index":1,"sense_key":"wonder%2:31:00::","synset_id":"wonder.v.01"},{"index":2,"sense_key":"wonder%2:32:12::","synset_id":"wonder.v.03"},{"index":3,"sense_key":"wonder%2:32:01::","synset_id":"wonder.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wonder%2:31:00::","call_ids":["senseval3.d000.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wonder%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":376,"cached_input_tokens":0,"output_tokens":155,"reasoning_output_tokens":139},"cost":{"currency":"USD","total_usd":0.006530000000000001,"input_uncached_usd":0.0018800000000000002,"input_cached_usd":0.0,"output_usd":0.0046500000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1487767500802875}
{"item_id":"senseval3.d000.s009.t001","gold_sense_keys":["get%2:30:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":3,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":4,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":5,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":6,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":7,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":8,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":9,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":10,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":11,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":12,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":13,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":14,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":15,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"},{"index":16,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":17,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":18,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":19,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":20,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":21,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":22,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":23,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":24,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":25,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":26,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":27,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":28,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":29,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":30,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":31,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":32,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":33,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":34,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":35,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":36,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"get%2:30:00::","call_ids":["senseval3.d000.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"get%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1881,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":126},"cost":{"currency":"USD","total_usd":0.013665,"input_uncached_usd":0.009405,"input_cached_usd":0.0,"output_usd":0.00426,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8318956249859184}
{"item_id":"senseval3.d000.s009.t003","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":310,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":121},"cost":{"currency":"USD","total_usd":0.00566,"input_uncached_usd":0.0015500000000000002,"input_cached_usd":0.0,"output_usd":0.00411,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0327448749449104}
{"item_id":"senseval3.d000.s010.t000","gold_sense_keys":["remember%2:31:00::"],"candidates":[{"index":1,"sense_key":"remember%2:31:09::","synset_id":"commemorate.v.02"},{"index":2,"sense_key":"remember%2:40:00::","synset_id":"remember.v.04"},{"index":3,"sense_key":"remember%2:31:08::","synset_id":"remember.v.07"},{"index":4,"sense_key":"remember%2:32:07::","synset_id":"remember.v.05"},{"index":5,"sense_key":"remember%2:32:00::","synset_id":"commend.v.05"},{"index":6,"sense_key":"remember%2:31:01::","synset_id":"remember.v.02"},{"index":7,"sense_key":"remember%2:31:00::","synset_id":"remember.v.01"},{"index":8,"sense_key":"remember%2:31:02::","synset_id":"remember.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"remember%2:31:00::","call_ids":["senseval3.d000.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"remember%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":88},"cost":{"currency":"USD","total_usd":0.006145,"input_uncached_usd":0.0030250000000000003,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.329330625012517}
{"item_id":"senseval3.d000.s011.t001","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"},{"index":2,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"},{"index":3,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"},{"index":4,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"hour%1:28:00::","call_ids":["senseval3.d000.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":412,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.0038000000000000004,"input_uncached_usd":0.00206,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3853579169372097}
{"item_id":"senseval3.d000.s011.t002","gold_sense_keys":["steady%3:00:00::"],"candidates":[{"index":1,"sense_key":"steady%5:00:00:resolute:00","synset_id":"firm.s.01"},{"index":2,"sense_key":"steady%3:00:00::","synset_id":"steady.a.01"},{"index":3,"sense_key":"steady%5:00:00:stable:00","synset_id":"firm.s.06"},{"index":4,"sense_key":"steady%5:00:00:frequent:00","synset_id":"regular.s.06"},{"index":5,"sense_key":"steady%5:00:00:secure:03","synset_id":"steady.s.02"},{"index":6,"sense_key":"steady%5:00:00:unexcitable:00","synset_id":"steady.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"steady%3:00:00::","call_ids":["senseval3.d000.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"steady%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":119},"cost":{"currency":"USD","total_usd":0.006580000000000001,"input_uncached_usd":0.00253,"input_cached_usd":0.0,"output_usd":0.00405,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2081419579917565}
{"item_id":"senseval3.d000.s011.t003","gold_sense_keys":["drinking%1:04:01::"],"candidates":[{"index":1,"sense_key":"drinking%1:04:00::","synset_id":"drinking.n.01"},{"index":2,"sense_key":"drinking%1:04:01::","synset_id":"drink.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"drinking%1:04:01::","call_ids":["senseval3.d000.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"drinking%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":335,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":136},"cost":{"currency":"USD","total_usd":0.006235000000000001,"input_uncached_usd":0.001675,"input_cached_usd":0.0,"output_usd":0.00456,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.949190000072122}
{"item_id":"senseval3.d000.s012.t000","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":325,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.004085,"input_uncached_usd":0.0016250000000000001,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4263985831057653}
{"item_id":"senseval3.d000.s012.t001","gold_sense_keys":["head%1:08:00::"],"candidates":[{"index":1,"sense_key":"head%1:10:00::","synset_id":"heading.n.01"},{"index":2,"sense_key":"head%1:19:00::","synset_id":"head.n.06"},{"index":3,"sense_key":"head%1:18:02::","synset_id":"head.n.15"},{"index":4,"sense_key":"head%1:10:03::","synset_id":"head.n.09"},{"index":5,"sense_key":"head%1:06:03::","synset_id":"head.n.29"},{"index":6,"sense_key":"head%1:15:01::","synset_id":"fountainhead.n.02"},{"index":7,"sense_key":"head%1:14:00::","synset_id":"head.n.05"},{"index":8,"sense_key":"head%1:06:01::","synset_id":"drumhead.n.01"},{"index":9,"sense_key":"head%1:26:00::","synset_id":"head.n.10"},{"index":10,"sense_key":"head%1:08:01::","synset_id":"head.n.25"},{"index":11,"sense_key":"head%1:17:02::","synset_id":"promontory.n.01"},{"index":12,"sense_key":"head%1:17:01::","synset_id":"head.n.17"},{"index":13,"sense_key":"head%1:06:02::","synset_id":"head.n.30"},{"index":14,"sense_key":"head%1:11:01::","synset_id":"pass.n.09"},{"index":15,"sense_key":"head%1:06:00::","synset_id":"head.n.31"},{"index":16,"sense_key":"head%1:05:00::","synset_id":"head.n.02"},{"index":17,"sense_key":"head%1:10:02::","synset_id":"question.n.02"},{"index":18,"sense_key":"head%1:06:05::","synset_id":"read/write_head.n.01"},{"index":19,"sense_key":"head%1:23:00::","synset_id":"head.n.11"},{"index":20,"sense_key":"head%1:18:01::","synset_id":"principal.n.02"},{"index":21,"sense_key":"head%1:18:00::","synset_id":"head.n.04"},{"index":22,"sense_key":"head%1:08:00::","synset_id":"head.n.01"},{"index":23,"sense_key":"head%1:10:01::","synset_id":"point.n.20"},{"index":24,"sense_key":"head%1:08:02::","synset_id":"head.n.26"},{"index":25,"sense_key":"head%1:15:00::","synset_id":"head.n.07"},{"index":26,"sense_key":"head%1:20:00::","synset_id":"capitulum.n.01"},{"index":27,"sense_key":"head%1:18:03::","synset_id":"head.n.14"},{"index":28,"sense_key":"head%1:17:00::","synset_id":"head.n.18"},{"index":29,"sense_key":"head%1:15:02::","synset_id":"forefront.n.01"},{"index":30,"sense_key":"head%1:09:00::","synset_id":"mind.n.01"},{"index":31,"sense_key":"head%1:06:04::","synset_id":"head.n.28"},{"index":32,"sense_key":"head%1:11:00::","synset_id":"headway.n.02"},{"index":33,"sense_key":"head%1:04:00::","synset_id":"oral_sex.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":22,"chosen_sense_key":"head%1:08:00::","call_ids":["senseval3.d000.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":22,"predicted_sense_key":"head%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1547,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.023545000000000003,"input_uncached_usd":0.0077350000000000006,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.537920834030956}
{"item_id":"senseval3.d000.s012.t003","gold_sense_keys":["thought%1:09:00::"],"candidates":[{"index":1,"sense_key":"thought%1:09:01::","synset_id":"idea.n.01"},{"index":2,"sense_key":"thought%1:09:03::","synset_id":"opinion.n.01"},{"index":3,"sense_key":"thought%1:09:02::","synset_id":"thought.n.03"},{"index":4,"sense_key":"thought%1:09:00::","synset_id":"thinking.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"thought%1:09:00::","call_ids":["senseval3.d000.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"thought%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.004015,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.081925166072324}
{"item_id":"senseval3.d000.s012.t004","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":2,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":3,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":4,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":5,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":6,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":9,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":10,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":11,"sense_key":"start%2:38:03::","synset_id":"start.v.14"},{"index":12,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":13,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":14,"sense_key":"start%2:36:01::","synset_id":"start.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"start%2:30:00::","call_ids":["senseval3.d000.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"start%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":890,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":88},"cost":{"currency":"USD","total_usd":0.00757,"input_uncached_usd":0.00445,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.390856417012401}
{"item_id":"senseval3.d000.s012.t005","gold_sense_keys":["cross%2:38:00::"],"candidates":[{"index":1,"sense_key":"cross%2:38:00::","synset_id":"traverse.v.01"},{"index":2,"sense_key":"cross%2:38:03::","synset_id":"intersect.v.01"},{"index":3,"sense_key":"cross%2:36:09::","synset_id":"cross.v.07"},{"index":4,"sense_key":"cross%2:41:00::","synset_id":"thwart.v.01"},{"index":5,"sense_key":"cross%2:42:00::","synset_id":"cross.v.05"},{"index":6,"sense_key":"cross%2:38:04::","synset_id":"cross.v.04"},{"index":7,"sense_key":"cross%2:35:00::","synset_id":"crossbreed.v.01"},{"index":8,"sense_key":"cross%2:38:01::","synset_id":"cross.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cross%2:38:00::","call_ids":["senseval3.d000.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cross%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.00465,"input_uncached_usd":0.00303,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":65.1980794579722}
{"item_id":"senseval3.d000.s012.t006","gold_sense_keys":["street%1:06:01::"],"candidates":[{"index":1,"sense_key":"street%1:06:01::","synset_id":"street.n.02"},{"index":2,"sense_key":"street%1:26:00::","synset_id":"street.n.03"},{"index":3,"sense_key":"street%1:26:01::","synset_id":"street.n.04"},{"index":4,"sense_key":"street%1:06:00::","synset_id":"street.n.01"},{"index":5,"sense_key":"street%1:14:00::","synset_id":"street.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"street%1:06:01::","call_ids":["senseval3.d000.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"street%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.00459,"input_uncached_usd":0.0024000000000000002,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6385224169353023}
{"item_id":"senseval3.d000.s012.t007","gold_sense_keys":["light%1:10:00::"],"candidates":[{"index":1,"sense_key":"light%1:06:00::","synset_id":"light.n.02"},{"index":2,"sense_key":"light%1:15:00::","synset_id":"light.n.05"},{"index":3,"sense_key":"light%1:07:01::","synset_id":"luminosity.n.01"},{"index":4,"sense_key":"light%1:19:00::","synset_id":"light.n.01"},{"index":5,"sense_key":"light%1:10:00::","synset_id":"light.n.14"},{"index":6,"sense_key":"light%1:16:00::","synset_id":"inner_light.n.01"},{"index":7,"sense_key":"light%1:09:01::","synset_id":"light.n.03"},{"index":8,"sense_key":"light%1:07:02::","synset_id":"sparkle.n.01"},{"index":9,"sense_key":"light%1:18:00::","synset_id":"light.n.08"},{"index":10,"sense_key":"light%1:26:00::","synset_id":"light.n.09"},{"index":11,"sense_key":"light%1:09:00::","synset_id":"light.n.10"},{"index":12,"sense_key":"light%1:07:00::","synset_id":"light.n.07"},{"index":13,"sense_key":"light%1:06:01::","synset_id":"lighter.n.02"},{"index":14,"sense_key":"light%1:26:01::","synset_id":"light.n.06"},{"index":15,"sense_key":"light%1:09:02::","synset_id":"light.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"light%1:10:00::","call_ids":["senseval3.d000.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"light%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":890,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.00685,"input_uncached_usd":0.00445,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2195402910001576}
{"item_id":"senseval3.d000.s012.t010","gold_sense_keys":["roar%2:32:01::"],"candidates":[{"index":1,"sense_key":"roar%2:32:01::","synset_id":"roar.v.01"},{"index":2,"sense_key":"roar%2:32:05::","synset_id":"bellow.v.02"},{"index":3,"sense_key":"roar%2:38:00::","synset_id":"roar.v.04"},{"index":4,"sense_key":"roar%2:29:00::","synset_id":"roar.v.06"},{"index":5,"sense_key":"roar%2:32:03::","synset_id":"thunder.v.02"},{"index":6,"sense_key":"roar%2:32:02::","synset_id":"howl.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"roar%2:32:01::","call_ids":["senseval3.d000.s012.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"roar%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":185,"reasoning_output_tokens":169},"cost":{"currency":"USD","total_usd":0.008235000000000001,"input_uncached_usd":0.0026850000000000003,"input_cached_usd":0.0,"output_usd":0.00555,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.516527583007701}
{"item_id":"senseval3.d000.s012.t011","gold_sense_keys":["intersection%1:06:00::"],"candidates":[{"index":1,"sense_key":"intersection%1:06:00::","synset_id":"intersection.n.02"},{"index":2,"sense_key":"intersection%1:09:01::","synset_id":"overlap.n.01"},{"index":3,"sense_key":"intersection%1:14:00::","synset_id":"intersection.n.04"},{"index":4,"sense_key":"intersection%1:09:00::","synset_id":"intersection.n.01"},{"index":5,"sense_key":"intersection%1:25:00::","synset_id":"intersection.n.03"},{"index":6,"sense_key":"intersection%1:04:00::","synset_id":"intersection.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"intersection%1:06:00::","call_ids":["senseval3.d000.s012.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"intersection%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.0038400000000000005,"input_uncached_usd":0.0024600000000000004,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4591289170784876}
{"item_id":"senseval3.d000.s013.t000","gold_sense_keys":["brake%1:06:00::"],"candidates":[{"index":1,"sense_key":"brake%1:06:00::","synset_id":"brake.n.01"},{"index":2,"sense_key":"brake%1:20:02::","synset_id":"brake.n.02"},{"index":3,"sense_key":"brake%1:20:01::","synset_id":"bracken.n.02"},{"index":4,"sense_key":"brake%1:14:00::","synset_id":"brake.n.04"},{"index":5,"sense_key":"brake%1:06:01::","synset_id":"brake.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brake%1:06:00::","call_ids":["senseval3.d000.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brake%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004145,"input_uncached_usd":0.0023750000000000004,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.492504500085488}
{"item_id":"senseval3.d000.s013.t001","gold_sense_keys":["howl%2:32:01::"],"candidates":[{"index":1,"sense_key":"howl%2:29:00::","synset_id":"roar.v.06"},{"index":2,"sense_key":"howl%2:32:02::","synset_id":"howl.v.01"},{"index":3,"sense_key":"howl%2:32:03::","synset_id":"howl.v.02"},{"index":4,"sense_key":"howl%2:32:01::","synset_id":"roar.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"howl%2:32:01::","call_ids":["senseval3.d000.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"howl%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.004645,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.488219999941066}
{"item_id":"senseval3.d000.s013.t002","gold_sense_keys":["horn%1:06:05::"],"candidates":[{"index":1,"sense_key":"horn%1:06:07::","synset_id":"horn.n.08"},{"index":2,"sense_key":"horn%1:10:02::","synset_id":"horn.n.03"},{"index":3,"sense_key":"horn%1:06:00::","synset_id":"french_horn.n.01"},{"index":4,"sense_key":"horn%1:06:01::","synset_id":"cornet.n.01"},{"index":5,"sense_key":"horn%1:27:00::","synset_id":"horn.n.07"},{"index":6,"sense_key":"horn%1:06:03::","synset_id":"horn.n.09"},{"index":7,"sense_key":"horn%1:05:01::","synset_id":"horn.n.02"},{"index":8,"sense_key":"horn%1:06:05::","synset_id":"automobile_horn.n.01"},{"index":9,"sense_key":"horn%1:06:06::","synset_id":"horn.n.01"},{"index":10,"sense_key":"horn%1:06:04::","synset_id":"horn.n.04"},{"index":11,"sense_key":"horn%1:05:02::","synset_id":"horn.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"horn%1:06:05::","call_ids":["senseval3.d000.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"horn%1:06:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0193,"input_uncached_usd":0.0034600000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.418664791970514}
{"item_id":"senseval3.d000.s013.t003","gold_sense_keys":["blare%2:39:00::"],"candidates":[{"index":1,"sense_key":"blare%2:39:01::","synset_id":"blast.v.01"},{"index":2,"sense_key":"blare%2:39:00::","synset_id":"honk.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"blare%2:39:00::","call_ids":["senseval3.d000.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"blare%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":362,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":96},"cost":{"currency":"USD","total_usd":0.00517,"input_uncached_usd":0.0018100000000000002,"input_cached_usd":0.0,"output_usd":0.00336,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4674297500168905}
{"item_id":"senseval3.d000.s013.t004","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":3,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":6,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":7,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":10,"sense_key":"man%1:05:00::","synset_id":"world.n.08"},{"index":11,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s013.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":815,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.006325000000000001,"input_uncached_usd":0.0040750000000000005,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8497523330152035}
{"item_id":"senseval3.d000.s013.t005","gold_sense_keys":["hit%2:35:01::"],"candidates":[{"index":1,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"},{"index":2,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"},{"index":3,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"},{"index":4,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":5,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":6,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"},{"index":7,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":8,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":9,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"},{"index":10,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":11,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":12,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":13,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":14,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":15,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":16,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"},{"index":17,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"hit%2:35:01::","call_ids":["senseval3.d000.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"hit%2:35:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1016,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.00751,"input_uncached_usd":0.00508,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5765707079553977}
{"item_id":"senseval3.d000.s013.t006","gold_sense_keys":["call_out%2:32:01::"],"candidates":[{"index":1,"sense_key":"call_out%2:32:00::","synset_id":"call_out.v.02"},{"index":2,"sense_key":"call_out%2:32:01::","synset_id":"exclaim.v.01"},{"index":3,"sense_key":"call_out%2:32:12::","synset_id":"call_out.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"call_out%2:32:01::","call_ids":["senseval3.d000.s013.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"call_out%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":88},"cost":{"currency":"USD","total_usd":0.0051,"input_uncached_usd":0.00198,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2555543748894706}
{"item_id":"senseval3.d000.s013.t007","gold_sense_keys":["second%1:28:00::","second%1:28:01::"],"candidates":[{"index":1,"sense_key":"second%1:10:00::","synset_id":"second.n.08"},{"index":2,"sense_key":"second%1:04:00::","synset_id":"second_base.n.02"},{"index":3,"sense_key":"second%1:28:02::","synset_id":"moment.n.01"},{"index":4,"sense_key":"second%1:28:01::","synset_id":"moment.n.02"},{"index":5,"sense_key":"second%1:06:01::","synset_id":"irregular.n.02"},{"index":6,"sense_key":"second%1:06:00::","synset_id":"second_gear.n.01"},{"index":7,"sense_key":"second%1:28:00::","synset_id":"second.n.01"},{"index":8,"sense_key":"second%1:23:00::","synset_id":"second.n.06"},{"index":9,"sense_key":"second%1:18:00::","synset_id":"second.n.07"},{"index":10,"sense_key":"second%1:24:00::","synset_id":"second.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"second%1:28:01::","call_ids":["senseval3.d000.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"second%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":358,"reasoning_output_tokens":342},"cost":{"currency":"USD","total_usd":0.014204999999999999,"input_uncached_usd":0.003465,"input_cached_usd":0.0,"output_usd":0.01074,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.648893708013929}
{"item_id":"senseval3.d000.s014.t002","gold_sense_keys":["do%2:36:01::"],"candidates":[{"index":1,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":2,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":3,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":4,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":5,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":6,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":7,"sense_key":"do%2:38:00::","synset_id":"do.v.13"},{"index":8,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":9,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":10,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":11,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":12,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":13,"sense_key":"do%2:29:09::","synset_id":"act.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"do%2:36:01::","call_ids":["senseval3.d000.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":887,"cached_input_tokens":0,"output_tokens":168,"reasoning_output_tokens":152},"cost":{"currency":"USD","total_usd":0.009475,"input_uncached_usd":0.004435000000000001,"input_cached_usd":0.0,"output_usd":0.00504,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.661996457958594}
{"item_id":"senseval3.d000.s015.t001","gold_sense_keys":["care%2:37:02::"],"candidates":[{"index":1,"sense_key":"care%2:41:13::","synset_id":"care.v.02"},{"index":2,"sense_key":"care%2:37:00::","synset_id":"wish.v.02"},{"index":3,"sense_key":"care%2:37:02::","synset_id":"care.v.01"},{"index":4,"sense_key":"care%2:37:12::","synset_id":"worry.v.02"},{"index":5,"sense_key":"care%2:41:11::","synset_id":"manage.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"care%2:37:02::","call_ids":["senseval3.d000.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"care%2:37:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.004315,"input_uncached_usd":0.002575,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6279539169045165}
{"item_id":"senseval3.d000.s015.t002","gold_sense_keys":["guy%1:18:00::"],"candidates":[{"index":1,"sense_key":"guy%1:06:00::","synset_id":"guy.n.03"},{"index":2,"sense_key":"guy%1:18:00::","synset_id":"guy.n.01"},{"index":3,"sense_key":"guy%1:06:01::","synset_id":"guy.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"guy%1:18:00::","call_ids":["senseval3.d000.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"guy%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.0038650000000000004,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.60289850004483}
{"item_id":"senseval3.d000.s015.t003","gold_sense_keys":["hit%2:35:01::"],"candidates":[{"index":1,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":2,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"},{"index":3,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":4,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":5,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":6,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"},{"index":7,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"},{"index":8,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":9,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"},{"index":10,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":11,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"},{"index":12,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":13,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":14,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"},{"index":15,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":16,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":17,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"hit%2:35:01::","call_ids":["senseval3.d000.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"hit%2:35:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1047,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":96},"cost":{"currency":"USD","total_usd":0.008565,"input_uncached_usd":0.005235,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0692957499995828}
{"item_id":"senseval3.d000.s016.t000","gold_sense_keys":["regret%2:37:00::"],"candidates":[{"index":1,"sense_key":"regret%2:37:01::","synset_id":"regret.v.02"},{"index":2,"sense_key":"regret%2:37:00::","synset_id":"repent.v.02"},{"index":3,"sense_key":"regret%2:32:02::","synset_id":"regret.v.04"},{"index":4,"sense_key":"regret%2:32:00::","synset_id":"regret.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"regret%2:37:00::","call_ids":["senseval3.d000.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"regret%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":463,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.005105,"input_uncached_usd":0.0023150000000000002,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.357130541931838}
{"item_id":"senseval3.d000.s016.t001","gold_sense_keys":["open%2:35:00::"],"candidates":[{"index":1,"sense_key":"open%2:35:08::","synset_id":"unfold.v.04"},{"index":2,"sense_key":"open%2:33:00::","synset_id":"open.v.09"},{"index":3,"sense_key":"open%2:30:01::","synset_id":"open.v.07"},{"index":4,"sense_key":"open%2:42:00::","synset_id":"open.v.08"},{"index":5,"sense_key":"open%2:39:10::","synset_id":"open.v.11"},{"index":6,"sense_key":"open%2:35:00::","synset_id":"open.v.01"},{"index":7,"sense_key":"open%2:30:00::","synset_id":"open.v.06"},{"index":8,"sense_key":"open%2:41:00::","synset_id":"open.v.02"},{"index":9,"sense_key":"open%2:35:06::","synset_id":"open.v.03"},{"index":10,"sense_key":"open%2:40:04::","synset_id":"afford.v.04"},{"index":11,"sense_key":"open%2:41:01::","synset_id":"open.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"open%2:35:00::","call_ids":["senseval3.d000.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"open%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":713,"cached_input_tokens":0,"output_tokens":214,"reasoning_output_tokens":198},"cost":{"currency":"USD","total_usd":0.009985,"input_uncached_usd":0.0035650000000000005,"input_cached_usd":0.0,"output_usd":0.00642,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.25967654096894}
{"item_id":"senseval3.d000.s016.t002","gold_sense_keys":["mouth%1:08:01::"],"candidates":[{"index":1,"sense_key":"mouth%1:18:01::","synset_id":"mouthpiece.n.03"},{"index":2,"sense_key":"mouth%1:10:00::","synset_id":"sass.n.01"},{"index":3,"sense_key":"mouth%1:17:00::","synset_id":"mouth.n.04"},{"index":4,"sense_key":"mouth%1:08:00::","synset_id":"mouth.n.02"},{"index":5,"sense_key":"mouth%1:08:01::","synset_id":"mouth.n.01"},{"index":6,"sense_key":"mouth%1:06:00::","synset_id":"mouth.n.08"},{"index":7,"sense_key":"mouth%1:18:00::","synset_id":"mouth.n.05"},{"index":8,"sense_key":"mouth%1:17:01::","synset_id":"mouth.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"mouth%1:08:01::","call_ids":["senseval3.d000.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"mouth%1:08:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":146},"cost":{"currency":"USD","total_usd":0.008165,"input_uncached_usd":0.003305,"input_cached_usd":0.0,"output_usd":0.00486,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.233763042022474}
{"item_id":"senseval3.d000.s016.t003","gold_sense_keys":["truck%1:06:00::"],"candidates":[{"index":1,"sense_key":"truck%1:06:00::","synset_id":"truck.n.01"},{"index":2,"sense_key":"truck%1:06:01::","synset_id":"hand_truck.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"truck%1:06:00::","call_ids":["senseval3.d000.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"truck%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.0039050000000000005,"input_uncached_usd":0.0020450000000000004,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5524728749878705}
{"item_id":"senseval3.d000.s016.t005","gold_sense_keys":["stop%1:11:00::"],"candidates":[{"index":1,"sense_key":"stop%1:06:03::","synset_id":"stop.n.08"},{"index":2,"sense_key":"stop%1:06:00::","synset_id":"catch.n.06"},{"index":3,"sense_key":"stop%1:11:00::","synset_id":"stop.n.01"},{"index":4,"sense_key":"stop%1:04:02::","synset_id":"stop.n.03"},{"index":5,"sense_key":"stop%1:04:00::","synset_id":"stop.n.02"},{"index":6,"sense_key":"stop%1:06:02::","synset_id":"diaphragm.n.01"},{"index":7,"sense_key":"stop%1:10:01::","synset_id":"period.n.07"},{"index":8,"sense_key":"stop%1:06:01::","synset_id":"blockage.n.02"},{"index":9,"sense_key":"stop%1:15:00::","synset_id":"stop.n.05"},{"index":10,"sense_key":"stop%1:10:00::","synset_id":"stop_consonant.n.01"},{"index":11,"sense_key":"stop%1:26:00::","synset_id":"arrest.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"stop%1:11:00::","call_ids":["senseval3.d000.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"stop%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":868,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.00638,"input_uncached_usd":0.00434,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5739167500287294}
{"item_id":"senseval3.d000.s016.t006","gold_sense_keys":["angry%3:00:00::"],"candidates":[{"index":1,"sense_key":"angry%5:00:00:unhealthy:00","synset_id":"angry.s.02"},{"index":2,"sense_key":"angry%5:00:00:stormy:00","synset_id":"angry.s.01"},{"index":3,"sense_key":"angry%3:00:00::","synset_id":"angry.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"angry%3:00:00::","call_ids":["senseval3.d000.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"angry%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.0038250000000000003,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3569436250254512}
{"item_id":"senseval3.d000.s016.t007","gold_sense_keys":["driver%1:18:00::"],"candidates":[{"index":1,"sense_key":"driver%1:06:00::","synset_id":"driver.n.05"},{"index":2,"sense_key":"driver%1:18:02::","synset_id":"driver.n.02"},{"index":3,"sense_key":"driver%1:18:00::","synset_id":"driver.n.01"},{"index":4,"sense_key":"driver%1:10:00::","synset_id":"driver.n.04"},{"index":5,"sense_key":"driver%1:18:01::","synset_id":"driver.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"driver%1:18:00::","call_ids":["senseval3.d000.s016.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"driver%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.004155000000000001,"input_uncached_usd":0.002415,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3165142089128494}
{"item_id":"senseval3.d000.s016.t008","gold_sense_keys":["jump%2:38:11::"],"candidates":[{"index":1,"sense_key":"jump%2:42:00::","synset_id":"leap_out.v.01"},{"index":2,"sense_key":"jump%2:38:11::","synset_id":"jump.v.08"},{"index":3,"sense_key":"jump%2:38:00::","synset_id":"jump.v.01"},{"index":4,"sense_key":"jump%2:33:03::","synset_id":"jump.v.06"},{"index":5,"sense_key":"jump%2:38:04::","synset_id":"startle.v.02"},{"index":6,"sense_key":"jump%2:38:01::","synset_id":"jump.v.11"},{"index":7,"sense_key":"jump%2:38:07::","synset_id":"jumpstart.v.01"},{"index":8,"sense_key":"jump%2:31:00::","synset_id":"jump.v.13"},{"index":9,"sense_key":"jump%2:30:00::","synset_id":"jump.v.04"},{"index":10,"sense_key":"jump%2:33:00::","synset_id":"jump.v.03"},{"index":11,"sense_key":"jump%2:38:05::","synset_id":"derail.v.02"},{"index":12,"sense_key":"jump%2:38:12::","synset_id":"chute.v.01"},{"index":13,"sense_key":"jump%2:30:05::","synset_id":"leap.v.02"},{"index":14,"sense_key":"jump%2:30:03::","synset_id":"alternate.v.01"},{"index":15,"sense_key":"jump%2:30:01::","synset_id":"rise.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"jump%2:38:11::","call_ids":["senseval3.d000.s016.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"jump%2:38:11::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":926,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.0064600000000000005,"input_uncached_usd":0.0046300000000000004,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.415925665991381}
{"item_id":"senseval3.d000.s016.t009","gold_sense_keys":["cab%1:06:02::"],"candidates":[{"index":1,"sense_key":"cab%1:06:01::","synset_id":"cab.n.02"},{"index":2,"sense_key":"cab%1:06:00::","synset_id":"cab.n.03"},{"index":3,"sense_key":"cab%1:06:02::","synset_id":"cab.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cab%1:06:02::","call_ids":["senseval3.d000.s016.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cab%1:06:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00402,"input_uncached_usd":0.00222,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3690750419627875}
{"item_id":"senseval3.d000.s016.t010","gold_sense_keys":["walk%2:38:00::"],"candidates":[{"index":1,"sense_key":"walk%2:38:00::","synset_id":"walk.v.01"},{"index":2,"sense_key":"walk%2:41:00::","synset_id":"walk.v.06"},{"index":3,"sense_key":"walk%2:38:02::","synset_id":"walk.v.08"},{"index":4,"sense_key":"walk%2:33:00::","synset_id":"walk.v.03"},{"index":5,"sense_key":"walk%2:33:01::","synset_id":"walk.v.05"},{"index":6,"sense_key":"walk%2:38:05::","synset_id":"walk.v.09"},{"index":7,"sense_key":"walk%2:38:04::","synset_id":"walk.v.10"},{"index":8,"sense_key":"walk%2:38:03::","synset_id":"walk.v.04"},{"index":9,"sense_key":"walk%2:41:01::","synset_id":"walk.v.07"},{"index":10,"sense_key":"walk%2:38:01::","synset_id":"walk.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"walk%2:38:00::","call_ids":["senseval3.d000.s016.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"walk%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":670,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.0059900000000000005,"input_uncached_usd":0.00335,"input_cached_usd":0.0,"output_usd":0.00264,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.788427958963439}
{"item_id":"senseval3.d000.s017.t000","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"},{"index":2,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.004229999999999999,"input_uncached_usd":0.00198,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0932177499635145}
{"item_id":"senseval3.d000.s017.t003","gold_sense_keys":["arm%1:08:00::"],"candidates":[{"index":1,"sense_key":"arm%1:14:00::","synset_id":"branch.n.01"},{"index":2,"sense_key":"arm%1:06:00::","synset_id":"sleeve.n.01"},{"index":3,"sense_key":"arm%1:06:02::","synset_id":"arm.n.04"},{"index":4,"sense_key":"arm%1:08:00::","synset_id":"arm.n.01"},{"index":5,"sense_key":"arm%1:06:01::","synset_id":"weapon.n.01"},{"index":6,"sense_key":"arm%1:06:03::","synset_id":"arm.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"arm%1:08:00::","call_ids":["senseval3.d000.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"arm%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.00446,"input_uncached_usd":0.0029600000000000004,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.352709832950495}
{"item_id":"senseval3.d000.s017.t004","gold_sense_keys":["shoulder%1:08:00::"],"candidates":[{"index":1,"sense_key":"shoulder%1:08:00::","synset_id":"shoulder.n.01"},{"index":2,"sense_key":"shoulder%1:08:02::","synset_id":"shoulder.n.03"},{"index":3,"sense_key":"shoulder%1:06:01::","synset_id":"shoulder.n.04"},{"index":4,"sense_key":"shoulder%1:06:00::","synset_id":"shoulder.n.05"},{"index":5,"sense_key":"shoulder%1:13:00::","synset_id":"shoulder.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"shoulder%1:08:00::","call_ids":["senseval3.d000.s017.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"shoulder%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.004345,"input_uncached_usd":0.0026650000000000003,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3346694589126855}
{"item_id":"senseval3.d000.s017.t005","gold_sense_keys":["old%3:00:01::"],"candidates":[{"index":1,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"},{"index":2,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":3,"sense_key":"old%5:00:00:preceding:00","synset_id":"previous.s.01"},{"index":4,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":5,"sense_key":"old%5:00:03:past:00","synset_id":"erstwhile.s.01"},{"index":6,"sense_key":"old%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":7,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"},{"index":8,"sense_key":"old%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"old%3:00:01::","call_ids":["senseval3.d000.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"old%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019375,"input_uncached_usd":0.0035350000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.175126582966186}
{"item_id":"senseval3.d000.s017.t006","gold_sense_keys":["friend%1:18:00::"],"candidates":[{"index":1,"sense_key":"friend%1:18:04::","synset_id":"supporter.n.01"},{"index":2,"sense_key":"friend%1:18:00::","synset_id":"friend.n.01"},{"index":3,"sense_key":"friend%1:18:02::","synset_id":"acquaintance.n.03"},{"index":4,"sense_key":"friend%1:18:01::","synset_id":"friend.n.05"},{"index":5,"sense_key":"friend%1:18:03::","synset_id":"ally.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"friend%1:18:00::","call_ids":["senseval3.d000.s017.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"friend%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.004625000000000001,"input_uncached_usd":0.002795,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8876535419840366}
{"item_id":"senseval3.d000.s018.t000","gold_sense_keys":["driver%1:18:00::"],"candidates":[{"index":1,"sense_key":"driver%1:10:00::","synset_id":"driver.n.04"},{"index":2,"sense_key":"driver%1:18:00::","synset_id":"driver.n.01"},{"index":3,"sense_key":"driver%1:06:00::","synset_id":"driver.n.05"},{"index":4,"sense_key":"driver%1:18:02::","synset_id":"driver.n.02"},{"index":5,"sense_key":"driver%1:18:01::","synset_id":"driver.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"driver%1:18:00::","call_ids":["senseval3.d000.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"driver%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.0040149999999999995,"input_uncached_usd":0.002395,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6420910420129076}
{"item_id":"senseval3.d000.s018.t001","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":2,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":3,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":4,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":5,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":6,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":7,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":8,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":9,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":10,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":11,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":12,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":13,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"start%2:30:00::","call_ids":["senseval3.d000.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"start%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":951,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":134},"cost":{"currency":"USD","total_usd":0.009255000000000001,"input_uncached_usd":0.0047550000000000005,"input_cached_usd":0.0,"output_usd":0.0045000000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2813042500056326}
{"item_id":"senseval3.d000.s018.t002","gold_sense_keys":["curse%2:32:02::"],"candidates":[{"index":1,"sense_key":"curse%2:32:02::","synset_id":"curse.v.02"},{"index":2,"sense_key":"curse%2:32:03::","synset_id":"excommunicate.v.01"},{"index":3,"sense_key":"curse%2:32:01::","synset_id":"curse.v.03"},{"index":4,"sense_key":"curse%2:32:00::","synset_id":"curse.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"curse%2:32:02::","call_ids":["senseval3.d000.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"curse%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.00554,"input_uncached_usd":0.0026000000000000003,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.628288083942607}
{"item_id":"senseval3.d000.s018.t003","gold_sense_keys":["plot%1:09:00::"],"candidates":[{"index":1,"sense_key":"plot%1:10:00::","synset_id":"plot.n.03"},{"index":2,"sense_key":"plot%1:10:01::","synset_id":"plot.n.04"},{"index":3,"sense_key":"plot%1:15:00::","synset_id":"plot.n.02"},{"index":4,"sense_key":"plot%1:09:00::","synset_id":"plot.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"plot%1:09:00::","call_ids":["senseval3.d000.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"plot%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.004195000000000001,"input_uncached_usd":0.0025150000000000003,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5248091249959543}
{"item_id":"senseval3.d000.s018.t004","gold_sense_keys":["ruin%2:41:00::"],"candidates":[{"index":1,"sense_key":"ruin%2:41:00::","synset_id":"ruin.v.02"},{"index":2,"sense_key":"ruin%2:35:02::","synset_id":"deflower.v.01"},{"index":3,"sense_key":"ruin%2:35:01::","synset_id":"ruin.v.04"},{"index":4,"sense_key":"ruin%2:30:00::","synset_id":"ruin.v.06"},{"index":5,"sense_key":"ruin%2:40:00::","synset_id":"bankrupt.v.01"},{"index":6,"sense_key":"ruin%2:35:00::","synset_id":"destroy.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ruin%2:41:00::","call_ids":["senseval3.d000.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ruin%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":561,"cached_input_tokens":0,"output_tokens":382,"reasoning_output_tokens":366},"cost":{"currency":"USD","total_usd":0.014265,"input_uncached_usd":0.002805,"input_cached_usd":0.0,"output_usd":0.01146,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.452302041929215}
{"item_id":"senseval3.d000.s018.t006","gold_sense_keys":["record%1:04:01::"],"candidates":[{"index":1,"sense_key":"record%1:04:01::","synset_id":"record.n.04"},{"index":2,"sense_key":"record%1:10:00::","synset_id":"criminal_record.n.01"},{"index":3,"sense_key":"record%1:23:00::","synset_id":"record.n.03"},{"index":4,"sense_key":"record%1:10:02::","synset_id":"record.n.05"},{"index":5,"sense_key":"record%1:21:00::","synset_id":"record.n.07"},{"index":6,"sense_key":"record%1:06:00::","synset_id":"phonograph_record.n.01"},{"index":7,"sense_key":"record%1:04:00::","synset_id":"record.n.06"},{"index":8,"sense_key":"record%1:10:03::","synset_id":"record.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"record%1:04:01::","call_ids":["senseval3.d000.s018.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"record%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":739,"cached_input_tokens":0,"output_tokens":213,"reasoning_output_tokens":198},"cost":{"currency":"USD","total_usd":0.010085,"input_uncached_usd":0.0036950000000000004,"input_cached_usd":0.0,"output_usd":0.00639,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.579054957954213}
{"item_id":"senseval3.d000.s019.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":2,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":3,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":4,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":7,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":8,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":9,"sense_key":"man%1:05:00::","synset_id":"world.n.08"},{"index":10,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":11,"sense_key":"man%1:18:00::","synset_id":"man.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.006845,"input_uncached_usd":0.0042650000000000006,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.051500124973245}
{"item_id":"senseval3.d000.s019.t001","gold_sense_keys":["save%2:41:00::"],"candidates":[{"index":1,"sense_key":"save%2:41:01::","synset_id":"salvage.v.01"},{"index":2,"sense_key":"save%2:40:01::","synset_id":"save.v.05"},{"index":3,"sense_key":"save%2:40:03::","synset_id":"keep_open.v.01"},{"index":4,"sense_key":"save%2:40:04::","synset_id":"save.v.09"},{"index":5,"sense_key":"save%2:32:04::","synset_id":"write.v.08"},{"index":6,"sense_key":"save%2:41:05::","synset_id":"spare.v.01"},{"index":7,"sense_key":"save%2:40:00::","synset_id":"save.v.02"},{"index":8,"sense_key":"save%2:41:00::","synset_id":"save.v.03"},{"index":9,"sense_key":"save%2:41:02::","synset_id":"deliver.v.08"},{"index":10,"sense_key":"save%2:30:00::","synset_id":"save.v.06"},{"index":11,"sense_key":"save%2:40:02::","synset_id":"save.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"save%2:41:01::","call_ids":["senseval3.d000.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"save%2:41:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019525,"input_uncached_usd":0.0036850000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.963729082955979}
{"item_id":"senseval3.d000.s019.t002","gold_sense_keys":["turn%2:38:00::"],"candidates":[{"index":1,"sense_key":"turn%2:36:00::","synset_id":"plow.v.01"},{"index":2,"sense_key":"turn%2:35:08::","synset_id":"turn.v.08"},{"index":3,"sense_key":"turn%2:38:02::","synset_id":"turn.v.04"},{"index":4,"sense_key":"turn%2:42:00::","synset_id":"become.v.02"},{"index":5,"sense_key":"turn%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":6,"sense_key":"turn%2:30:04::","synset_id":"change_state.v.01"},{"index":7,"sense_key":"turn%2:33:13::","synset_id":"turn.v.23"},{"index":8,"sense_key":"turn%2:35:06::","synset_id":"turn.v.13"},{"index":9,"sense_key":"turn%2:35:11::","synset_id":"flex.v.05"},{"index":10,"sense_key":"turn%2:30:09::","synset_id":"turn.v.14"},{"index":11,"sense_key":"turn%2:38:00::","synset_id":"turn.v.01"},{"index":12,"sense_key":"turn%2:35:10::","synset_id":"turn.v.22"},{"index":13,"sense_key":"turn%2:38:13::","synset_id":"turn.v.19"},{"index":14,"sense_key":"turn%2:29:00::","synset_id":"twist.v.10"},{"index":15,"sense_key":"turn%2:38:01::","synset_id":"turn.v.10"},{"index":16,"sense_key":"turn%2:40:13::","synset_id":"turn.v.18"},{"index":17,"sense_key":"turn%2:42:13::","synset_id":"turn.v.16"},{"index":18,"sense_key":"turn%2:30:14::","synset_id":"turn.v.26"},{"index":19,"sense_key":"turn%2:30:01::","synset_id":"sour.v.01"},{"index":20,"sense_key":"turn%2:41:10::","synset_id":"turn.v.17"},{"index":21,"sense_key":"turn%2:38:04::","synset_id":"turn.v.09"},{"index":22,"sense_key":"turn%2:38:03::","synset_id":"turn.v.06"},{"index":23,"sense_key":"turn%2:38:06::","synset_id":"turn.v.11"},{"index":24,"sense_key":"turn%2:30:03::","synset_id":"turn.v.07"},{"index":25,"sense_key":"turn%2:38:14::","synset_id":"turn.v.20"},{"index":26,"sense_key":"turn%2:32:14::","synset_id":"call_on.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"turn%2:38:00::","call_ids":["senseval3.d000.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"turn%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1440,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":111},"cost":{"currency":"USD","total_usd":0.01101,"input_uncached_usd":0.007200000000000001,"input_cached_usd":0.0,"output_usd":0.00381,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.480927708093077}
{"item_id":"senseval3.d000.s019.t003","gold_sense_keys":["look%2:39:00::"],"candidates":[{"index":1,"sense_key":"look%2:29:00::","synset_id":"look.v.03"},{"index":2,"sense_key":"look%2:39:00::","synset_id":"look.v.01"},{"index":3,"sense_key":"look%2:31:02::","synset_id":"count.v.08"},{"index":4,"sense_key":"look%2:39:01::","synset_id":"look.v.02"},{"index":5,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":6,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":7,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":8,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":9,"sense_key":"look%2:39:02::","synset_id":"search.v.02"},{"index":10,"sense_key":"look%2:42:00::","synset_id":"front.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"look%2:39:00::","call_ids":["senseval3.d000.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"look%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":760,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00572,"input_uncached_usd":0.0038000000000000004,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.784459250047803}
{"item_id":"senseval3.d000.s019.t005","gold_sense_keys":["face%1:08:00::"],"candidates":[{"index":1,"sense_key":"face%1:07:01::","synset_id":"face.n.11"},{"index":2,"sense_key":"face%1:08:00::","synset_id":"face.n.01"},{"index":3,"sense_key":"face%1:08:01::","synset_id":"face.n.07"},{"index":4,"sense_key":"face%1:10:00::","synset_id":"font.n.01"},{"index":5,"sense_key":"face%1:18:00::","synset_id":"face.n.05"},{"index":6,"sense_key":"face%1:15:00::","synset_id":"side.n.04"},{"index":7,"sense_key":"face%1:06:02::","synset_id":"face.n.04"},{"index":8,"sense_key":"face%1:10:01::","synset_id":"grimace.n.01"},{"index":9,"sense_key":"face%1:06:01::","synset_id":"face.n.08"},{"index":10,"sense_key":"face%1:07:00::","synset_id":"face.n.03"},{"index":11,"sense_key":"face%1:07:03::","synset_id":"expression.n.01"},{"index":12,"sense_key":"face%1:07:02::","synset_id":"boldness.n.02"},{"index":13,"sense_key":"face%1:06:00::","synset_id":"face.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"face%1:08:00::","call_ids":["senseval3.d000.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"face%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00606,"input_uncached_usd":0.00417,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.104680375079624}
{"item_id":"senseval3.d000.s019.t006","gold_sense_keys":["say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":2,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":3,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":6,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":7,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":8,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":9,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":10,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":11,"sense_key":"say%2:32:04::","synset_id":"order.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"say%2:32:15::","call_ids":["senseval3.d000.s019.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"say%2:32:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":799,"cached_input_tokens":0,"output_tokens":334,"reasoning_output_tokens":318},"cost":{"currency":"USD","total_usd":0.014015,"input_uncached_usd":0.003995,"input_cached_usd":0.0,"output_usd":0.01002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.08231266704388}
{"item_id":"senseval3.d000.s019.t007","gold_sense_keys":["word%1:10:03::"],"candidates":[{"index":1,"sense_key":"word%1:10:04::","synset_id":"discussion.n.02"},{"index":2,"sense_key":"word%1:18:00::","synset_id":"son.n.02"},{"index":3,"sense_key":"word%1:10:06::","synset_id":"word.n.04"},{"index":4,"sense_key":"word%1:10:03::","synset_id":"word.n.02"},{"index":5,"sense_key":"word%1:10:07::","synset_id":"bible.n.01"},{"index":6,"sense_key":"word%1:10:05::","synset_id":"password.n.01"},{"index":7,"sense_key":"word%1:10:02::","synset_id":"news.n.01"},{"index":8,"sense_key":"word%1:10:01::","synset_id":"parole.n.01"},{"index":9,"sense_key":"word%1:10:00::","synset_id":"word.n.01"},{"index":10,"sense_key":"word%1:23:00::","synset_id":"word.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"word%1:10:03::","call_ids":["senseval3.d000.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"word%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":171,"reasoning_output_tokens":155},"cost":{"currency":"USD","total_usd":0.008770000000000002,"input_uncached_usd":0.0036400000000000004,"input_cached_usd":0.0,"output_usd":0.00513,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.015264458023012}
{"item_id":"senseval3.d000.s020.t000","gold_sense_keys":["driver%1:18:00::"],"candidates":[{"index":1,"sense_key":"driver%1:18:02::","synset_id":"driver.n.02"},{"index":2,"sense_key":"driver%1:18:01::","synset_id":"driver.n.03"},{"index":3,"sense_key":"driver%1:10:00::","synset_id":"driver.n.04"},{"index":4,"sense_key":"driver%1:06:00::","synset_id":"driver.n.05"},{"index":5,"sense_key":"driver%1:18:00::","synset_id":"driver.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"driver%1:18:00::","call_ids":["senseval3.d000.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"driver%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.003980000000000001,"input_uncached_usd":0.0023000000000000004,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4543644579825923}
{"item_id":"senseval3.d000.s020.t001","gold_sense_keys":["stop%2:42:00::"],"candidates":[{"index":1,"sense_key":"stop%2:42:13::","synset_id":"end.v.01"},{"index":2,"sense_key":"stop%2:38:01::","synset_id":"stop.v.05"},{"index":3,"sense_key":"stop%2:42:00::","synset_id":"discontinue.v.01"},{"index":4,"sense_key":"stop%2:30:12::","synset_id":"hold_on.v.02"},{"index":5,"sense_key":"stop%2:38:02::","synset_id":"stop.v.04"},{"index":6,"sense_key":"stop%2:41:00::","synset_id":"stop.v.03"},{"index":7,"sense_key":"stop%2:35:00::","synset_id":"intercept.v.01"},{"index":8,"sense_key":"stop%2:35:12::","synset_id":"barricade.v.01"},{"index":9,"sense_key":"stop%2:30:05::","synset_id":"break.v.10"},{"index":10,"sense_key":"stop%2:38:00::","synset_id":"stop.v.01"},{"index":11,"sense_key":"stop%2:33:00::","synset_id":"check.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"stop%2:42:00::","call_ids":["senseval3.d000.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"stop%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":824,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":139},"cost":{"currency":"USD","total_usd":0.008740000000000001,"input_uncached_usd":0.00412,"input_cached_usd":0.0,"output_usd":0.00462,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2219099999638274}
{"item_id":"senseval3.d000.s020.t002","gold_sense_keys":["swear%2:32:01::"],"candidates":[{"index":1,"sense_key":"swear%2:32:03::","synset_id":"affirm.v.02"},{"index":2,"sense_key":"swear%2:31:10::","synset_id":"trust.v.01"},{"index":3,"sense_key":"swear%2:32:00::","synset_id":"swear.v.04"},{"index":4,"sense_key":"swear%2:32:02::","synset_id":"swear.v.03"},{"index":5,"sense_key":"swear%2:32:01::","synset_id":"curse.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"swear%2:32:01::","call_ids":["senseval3.d000.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"swear%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.004115,"input_uncached_usd":0.002585,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5903209580574185}
{"item_id":"senseval3.d000.s020.t003","gold_sense_keys":["turn%2:38:00::"],"candidates":[{"index":1,"sense_key":"turn%2:29:00::","synset_id":"twist.v.10"},{"index":2,"sense_key":"turn%2:35:06::","synset_id":"turn.v.13"},{"index":3,"sense_key":"turn%2:41:10::","synset_id":"turn.v.17"},{"index":4,"sense_key":"turn%2:32:14::","synset_id":"call_on.v.01"},{"index":5,"sense_key":"turn%2:38:06::","synset_id":"turn.v.11"},{"index":6,"sense_key":"turn%2:30:14::","synset_id":"turn.v.26"},{"index":7,"sense_key":"turn%2:38:13::","synset_id":"turn.v.19"},{"index":8,"sense_key":"turn%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":9,"sense_key":"turn%2:38:14::","synset_id":"turn.v.20"},{"index":10,"sense_key":"turn%2:36:00::","synset_id":"plow.v.01"},{"index":11,"sense_key":"turn%2:42:00::","synset_id":"become.v.02"},{"index":12,"sense_key":"turn%2:40:13::","synset_id":"turn.v.18"},{"index":13,"sense_key":"turn%2:38:01::","synset_id":"turn.v.10"},{"index":14,"sense_key":"turn%2:30:01::","synset_id":"sour.v.01"},{"index":15,"sense_key":"turn%2:38:00::","synset_id":"turn.v.01"},{"index":16,"sense_key":"turn%2:30:03::","synset_id":"turn.v.07"},{"index":17,"sense_key":"turn%2:35:11::","synset_id":"flex.v.05"},{"index":18,"sense_key":"turn%2:42:13::","synset_id":"turn.v.16"},{"index":19,"sense_key":"turn%2:30:09::","synset_id":"turn.v.14"},{"index":20,"sense_key":"turn%2:30:04::","synset_id":"change_state.v.01"},{"index":21,"sense_key":"turn%2:38:02::","synset_id":"turn.v.04"},{"index":22,"sense_key":"turn%2:38:03::","synset_id":"turn.v.06"},{"index":23,"sense_key":"turn%2:35:10::","synset_id":"turn.v.22"},{"index":24,"sense_key":"turn%2:33:13::","synset_id":"turn.v.23"},{"index":25,"sense_key":"turn%2:35:08::","synset_id":"turn.v.08"},{"index":26,"sense_key":"turn%2:38:04::","synset_id":"turn.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"turn%2:38:00::","call_ids":["senseval3.d000.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"turn%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1429,"cached_input_tokens":0,"output_tokens":299,"reasoning_output_tokens":284},"cost":{"currency":"USD","total_usd":0.016115,"input_uncached_usd":0.007145,"input_cached_usd":0.0,"output_usd":0.00897,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.861278541968204}
{"item_id":"senseval3.d000.s020.t004","gold_sense_keys":["heel%1:08:00::"],"candidates":[{"index":1,"sense_key":"heel%1:08:00::","synset_id":"heel.n.02"},{"index":2,"sense_key":"heel%1:06:00::","synset_id":"heel.n.01"},{"index":3,"sense_key":"heel%1:18:00::","synset_id":"cad.n.01"},{"index":4,"sense_key":"heel%1:13:00::","synset_id":"heel.n.04"},{"index":5,"sense_key":"heel%1:06:02::","synset_id":"heel.n.06"},{"index":6,"sense_key":"heel%1:06:03::","synset_id":"heel.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"heel%1:08:00::","call_ids":["senseval3.d000.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"heel%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":131},"cost":{"currency":"USD","total_usd":0.006895000000000001,"input_uncached_usd":0.0024850000000000002,"input_cached_usd":0.0,"output_usd":0.00441,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8616549590369686}
{"item_id":"senseval3.d000.s020.t005","gold_sense_keys":["truck%1:06:00::"],"candidates":[{"index":1,"sense_key":"truck%1:06:01::","synset_id":"hand_truck.n.01"},{"index":2,"sense_key":"truck%1:06:00::","synset_id":"truck.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"truck%1:06:00::","call_ids":["senseval3.d000.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"truck%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":385,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.003785,"input_uncached_usd":0.001925,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.647444041911513}
{"item_id":"senseval3.d000.s021.t000","gold_sense_keys":["give%2:31:02::"],"candidates":[{"index":1,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":2,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":3,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":4,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":5,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":6,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":7,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":8,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":9,"sense_key":"give%2:29:10::","synset_id":"give.v.44"},{"index":10,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":11,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":12,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":13,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":14,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":15,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":16,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":17,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":18,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":19,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":20,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":21,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":22,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":23,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":24,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":25,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":26,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":27,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":28,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":29,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":30,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":31,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":32,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":33,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":34,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":35,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":36,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":37,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":38,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":39,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":40,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":41,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":44,"sense_key":"give%2:40:13::","synset_id":"give.v.19"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"give%2:31:02::","call_ids":["senseval3.d000.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"give%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2059,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.012725,"input_uncached_usd":0.010295,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.388343417085707}
{"item_id":"senseval3.d000.s021.t002","gold_sense_keys":["thought%1:09:00::"],"candidates":[{"index":1,"sense_key":"thought%1:09:01::","synset_id":"idea.n.01"},{"index":2,"sense_key":"thought%1:09:03::","synset_id":"opinion.n.01"},{"index":3,"sense_key":"thought%1:09:00::","synset_id":"thinking.n.01"},{"index":4,"sense_key":"thought%1:09:02::","synset_id":"thought.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"thought%1:09:00::","call_ids":["senseval3.d000.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"thought%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.0047350000000000005,"input_uncached_usd":0.0025450000000000004,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4152336670085788}
{"item_id":"senseval3.d000.s022.t000","gold_sense_keys":["recall%2:31:00::"],"candidates":[{"index":1,"sense_key":"recall%2:31:00::","synset_id":"remember.v.01"},{"index":2,"sense_key":"recall%2:31:06::","synset_id":"recall.v.05"},{"index":3,"sense_key":"recall%2:41:00::","synset_id":"recall.v.06"},{"index":4,"sense_key":"recall%2:40:00::","synset_id":"recall.v.07"},{"index":5,"sense_key":"recall%2:32:02::","synset_id":"recall.v.04"},{"index":6,"sense_key":"recall%2:32:00::","synset_id":"hark_back.v.01"},{"index":7,"sense_key":"recall%2:42:00::","synset_id":"echo.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recall%2:31:00::","call_ids":["senseval3.d000.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recall%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.00483,"input_uncached_usd":0.00318,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9795955830486491}
{"item_id":"senseval3.d000.s022.t001","gold_sense_keys":["wonder%2:31:00::"],"candidates":[{"index":1,"sense_key":"wonder%2:31:00::","synset_id":"wonder.v.01"},{"index":2,"sense_key":"wonder%2:32:12::","synset_id":"wonder.v.03"},{"index":3,"sense_key":"wonder%2:32:01::","synset_id":"wonder.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wonder%2:31:00::","call_ids":["senseval3.d000.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wonder%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":425,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":108},"cost":{"currency":"USD","total_usd":0.005815000000000001,"input_uncached_usd":0.002125,"input_cached_usd":0.0,"output_usd":0.00369,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3447006249334663}
{"item_id":"senseval3.d000.s022.t002","gold_sense_keys":["see%2:36:00::","see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":2,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":3,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":4,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":9,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":10,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":11,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":12,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"},{"index":13,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":14,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":15,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":16,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":17,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":18,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":19,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":20,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":21,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":22,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:41:12::","synset_id":"see.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":21,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval3.d000.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":21,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1441,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.023045,"input_uncached_usd":0.0072050000000000005,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.57106016704347}
{"item_id":"senseval3.d000.s022.t003","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":3,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"},{"index":4,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":5,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":105},"cost":{"currency":"USD","total_usd":0.0061,"input_uncached_usd":0.0024700000000000004,"input_cached_usd":0.0,"output_usd":0.00363,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7980358749628067}
{"item_id":"senseval3.d000.s022.t004","gold_sense_keys":["make%2:32:00::","make%2:36:15::"],"candidates":[{"index":1,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":2,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":3,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":4,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":5,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":6,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":7,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":8,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":9,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":10,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":11,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":12,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":15,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":16,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":17,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":18,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":19,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":20,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":21,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":22,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":23,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":24,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":25,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":26,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":27,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":28,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":31,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":32,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":33,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":34,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":35,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":36,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":37,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":38,"sense_key":"make%2:29:01::","synset_id":"make.v.49"},{"index":39,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":40,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":41,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":42,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":43,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":44,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":45,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":46,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":47,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":48,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":49,"sense_key":"make%2:30:14::","synset_id":"make.v.46"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":24,"chosen_sense_key":"make%2:36:15::","call_ids":["senseval3.d000.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":24,"predicted_sense_key":"make%2:36:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2254,"cached_input_tokens":0,"output_tokens":7253,"reasoning_output_tokens":7158},"cost":{"currency":"USD","total_usd":0.22886,"input_uncached_usd":0.01127,"input_cached_usd":0.0,"output_usd":0.21759,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":121.06241020897869}
{"item_id":"senseval3.d000.s023.t000","gold_sense_keys":["sort%1:09:00::"],"candidates":[{"index":1,"sense_key":"sort%1:09:00::","synset_id":"kind.n.01"},{"index":2,"sense_key":"sort%1:22:00::","synset_id":"sort.n.04"},{"index":3,"sense_key":"sort%1:07:00::","synset_id":"sort.n.02"},{"index":4,"sense_key":"sort%1:18:00::","synset_id":"sort.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sort%1:09:00::","call_ids":["senseval3.d000.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sort%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":105},"cost":{"currency":"USD","total_usd":0.005955,"input_uncached_usd":0.0023250000000000002,"input_cached_usd":0.0,"output_usd":0.00363,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7322862499859184}
{"item_id":"senseval3.d000.s023.t001","gold_sense_keys":["look%1:07:01::"],"candidates":[{"index":1,"sense_key":"look%1:07:01::","synset_id":"expression.n.01"},{"index":2,"sense_key":"look%1:26:00::","synset_id":"spirit.n.02"},{"index":3,"sense_key":"look%1:07:00::","synset_id":"look.n.03"},{"index":4,"sense_key":"look%1:04:00::","synset_id":"look.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"look%1:07:01::","call_ids":["senseval3.d000.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"look%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":181,"reasoning_output_tokens":165},"cost":{"currency":"USD","total_usd":0.007845000000000001,"input_uncached_usd":0.002415,"input_cached_usd":0.0,"output_usd":0.00543,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8837780420435593}
{"item_id":"senseval3.d000.s023.t003","gold_sense_keys":["bluff%1:10:00::"],"candidates":[{"index":1,"sense_key":"bluff%1:10:00::","synset_id":"bluff.n.02"},{"index":2,"sense_key":"bluff%1:17:00::","synset_id":"bluff.n.01"},{"index":3,"sense_key":"bluff%1:04:00::","synset_id":"bluff.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bluff%1:10:00::","call_ids":["senseval3.d000.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bluff%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.00519,"input_uncached_usd":0.00198,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.672784042079002}
{"item_id":"senseval3.d000.s023.t004","gold_sense_keys":["say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":4,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":5,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":6,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":9,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":10,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":11,"sense_key":"say%2:32:13::","synset_id":"say.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"say%2:32:15::","call_ids":["senseval3.d000.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"say%2:32:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":338,"reasoning_output_tokens":322},"cost":{"currency":"USD","total_usd":0.013975,"input_uncached_usd":0.0038350000000000003,"input_cached_usd":0.0,"output_usd":0.01014,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.419374290970154}
{"item_id":"senseval3.d000.s023.t005","gold_sense_keys":["word%1:10:03::"],"candidates":[{"index":1,"sense_key":"word%1:23:00::","synset_id":"word.n.07"},{"index":2,"sense_key":"word%1:10:04::","synset_id":"discussion.n.02"},{"index":3,"sense_key":"word%1:10:02::","synset_id":"news.n.01"},{"index":4,"sense_key":"word%1:10:06::","synset_id":"word.n.04"},{"index":5,"sense_key":"word%1:10:07::","synset_id":"bible.n.01"},{"index":6,"sense_key":"word%1:10:00::","synset_id":"word.n.01"},{"index":7,"sense_key":"word%1:18:00::","synset_id":"son.n.02"},{"index":8,"sense_key":"word%1:10:03::","synset_id":"word.n.02"},{"index":9,"sense_key":"word%1:10:05::","synset_id":"password.n.01"},{"index":10,"sense_key":"word%1:10:01::","synset_id":"parole.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"word%1:10:03::","call_ids":["senseval3.d000.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"word%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":696,"cached_input_tokens":0,"output_tokens":213,"reasoning_output_tokens":197},"cost":{"currency":"USD","total_usd":0.00987,"input_uncached_usd":0.0034800000000000005,"input_cached_usd":0.0,"output_usd":0.00639,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2481894169468433}
{"item_id":"senseval3.d000.s024.t000","gold_sense_keys":["light%1:10:00::"],"candidates":[{"index":1,"sense_key":"light%1:26:01::","synset_id":"light.n.06"},{"index":2,"sense_key":"light%1:07:01::","synset_id":"luminosity.n.01"},{"index":3,"sense_key":"light%1:19:00::","synset_id":"light.n.01"},{"index":4,"sense_key":"light%1:16:00::","synset_id":"inner_light.n.01"},{"index":5,"sense_key":"light%1:10:00::","synset_id":"light.n.14"},{"index":6,"sense_key":"light%1:09:01::","synset_id":"light.n.03"},{"index":7,"sense_key":"light%1:18:00::","synset_id":"light.n.08"},{"index":8,"sense_key":"light%1:26:00::","synset_id":"light.n.09"},{"index":9,"sense_key":"light%1:06:01::","synset_id":"lighter.n.02"},{"index":10,"sense_key":"light%1:15:00::","synset_id":"light.n.05"},{"index":11,"sense_key":"light%1:06:00::","synset_id":"light.n.02"},{"index":12,"sense_key":"light%1:07:02::","synset_id":"sparkle.n.01"},{"index":13,"sense_key":"light%1:07:00::","synset_id":"light.n.07"},{"index":14,"sense_key":"light%1:09:00::","synset_id":"light.n.10"},{"index":15,"sense_key":"light%1:09:02::","synset_id":"light.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"light%1:10:00::","call_ids":["senseval3.d000.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"light%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":916,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":95},"cost":{"currency":"USD","total_usd":0.007880000000000002,"input_uncached_usd":0.004580000000000001,"input_cached_usd":0.0,"output_usd":0.0033,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.366937333950773}
{"item_id":"senseval3.d000.s024.t002","gold_sense_keys":["way%1:15:00::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:21:00::","synset_id":"way.n.12"},{"index":3,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":4,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":5,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":6,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":7,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":8,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":9,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":10,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":11,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":12,"sense_key":"way%1:04:00::","synset_id":"way.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"way%1:09:00::","call_ids":["senseval3.d000.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"way%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":759,"cached_input_tokens":0,"output_tokens":2082,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.06625500000000001,"input_uncached_usd":0.003795,"input_cached_usd":0.0,"output_usd":0.06246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":33.25526725000236}
{"item_id":"senseval3.d000.s024.t003","gold_sense_keys":["go_on%2:38:00::"],"candidates":[{"index":1,"sense_key":"go_on%2:38:00::","synset_id":"advance.v.01"},{"index":2,"sense_key":"go_on%2:42:00::","synset_id":"continue.v.01"},{"index":3,"sense_key":"go_on%2:35:00::","synset_id":"go_on.v.05"},{"index":4,"sense_key":"go_on%2:30:00::","synset_id":"happen.v.01"},{"index":5,"sense_key":"go_on%2:32:00::","synset_id":"continue.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"go_on%2:38:00::","call_ids":["senseval3.d000.s024.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"go_on%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":218,"reasoning_output_tokens":202},"cost":{"currency":"USD","total_usd":0.009225,"input_uncached_usd":0.0026850000000000003,"input_cached_usd":0.0,"output_usd":0.00654,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.330544500029646}
{"item_id":"senseval3.d000.s024.t004","gold_sense_keys":["street%1:06:01::"],"candidates":[{"index":1,"sense_key":"street%1:26:01::","synset_id":"street.n.04"},{"index":2,"sense_key":"street%1:14:00::","synset_id":"street.n.05"},{"index":3,"sense_key":"street%1:06:00::","synset_id":"street.n.01"},{"index":4,"sense_key":"street%1:06:01::","synset_id":"street.n.02"},{"index":5,"sense_key":"street%1:26:00::","synset_id":"street.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"street%1:06:01::","call_ids":["senseval3.d000.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"street%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":93},"cost":{"currency":"USD","total_usd":0.005795,"input_uncached_usd":0.0025250000000000003,"input_cached_usd":0.0,"output_usd":0.00327,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3117080840747803}
{"item_id":"senseval3.d000.s025.t000","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":355,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.004775,"input_uncached_usd":0.001775,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.904946624999866}
{"item_id":"senseval3.d000.s025.t001","gold_sense_keys":["find_out%2:31:00::"],"candidates":[{"index":1,"sense_key":"find_out%2:31:00::","synset_id":"learn.v.02"},{"index":2,"sense_key":"find_out%2:32:00::","synset_id":"determine.v.01"},{"index":3,"sense_key":"find_out%2:31:01::","synset_id":"find_out.v.04"},{"index":4,"sense_key":"find_out%2:32:01::","synset_id":"determine.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"find_out%2:31:00::","call_ids":["senseval3.d000.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"find_out%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":539,"cached_input_tokens":0,"output_tokens":430,"reasoning_output_tokens":414},"cost":{"currency":"USD","total_usd":0.015595000000000001,"input_uncached_usd":0.0026950000000000003,"input_cached_usd":0.0,"output_usd":0.0129,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.058546125073917}
{"item_id":"senseval3.d000.s025.t003","gold_sense_keys":["favorite%5:00:00:loved:00"],"candidates":[{"index":1,"sense_key":"favorite%5:00:00:loved:00","synset_id":"favored.s.01"},{"index":2,"sense_key":"favorite%5:00:00:popular:00","synset_id":"favorite.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"favorite%5:00:00:loved:00","call_ids":["senseval3.d000.s025.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"favorite%5:00:00:loved:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":379,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.004325,"input_uncached_usd":0.0018950000000000002,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.3430337079335}
{"item_id":"senseval3.d000.s025.t004","gold_sense_keys":["bar%1:06:04::"],"candidates":[{"index":1,"sense_key":"bar%1:06:01::","synset_id":"browning_automatic_rifle.n.01"},{"index":2,"sense_key":"bar%1:07:00::","synset_id":"stripe.n.05"},{"index":3,"sense_key":"bar%1:14:00::","synset_id":"legal_profession.n.01"},{"index":4,"sense_key":"bar%1:06:02::","synset_id":"bar.n.05"},{"index":5,"sense_key":"bar%1:06:04::","synset_id":"barroom.n.01"},{"index":6,"sense_key":"bar%1:06:06::","synset_id":"cake.n.01"},{"index":7,"sense_key":"bar%1:06:05::","synset_id":"bar.n.02"},{"index":8,"sense_key":"bar%1:17:00::","synset_id":"bar.n.08"},{"index":9,"sense_key":"bar%1:06:08::","synset_id":"bar.n.14"},{"index":10,"sense_key":"bar%1:06:00::","synset_id":"bar.n.03"},{"index":11,"sense_key":"bar%1:10:00::","synset_id":"measure.n.07"},{"index":12,"sense_key":"bar%1:23:00::","synset_id":"bar.n.07"},{"index":13,"sense_key":"bar%1:06:07::","synset_id":"bar.n.15"},{"index":14,"sense_key":"bar%1:04:00::","synset_id":"prevention.n.01"},{"index":15,"sense_key":"bar%1:06:09::","synset_id":"bar.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"bar%1:06:04::","call_ids":["senseval3.d000.s025.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"bar%1:06:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":978,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.00642,"input_uncached_usd":0.00489,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7791068750666454}
{"item_id":"senseval3.d000.s025.t005","gold_sense_keys":["insist%2:32:00::"],"candidates":[{"index":1,"sense_key":"insist%2:32:03::","synset_id":"importune.v.01"},{"index":2,"sense_key":"insist%2:31:00::","synset_id":"insist.v.03"},{"index":3,"sense_key":"insist%2:32:00::","synset_id":"insist.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"insist%2:32:00::","call_ids":["senseval3.d000.s025.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"insist%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":408,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.00558,"input_uncached_usd":0.00204,"input_cached_usd":0.0,"output_usd":0.00354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.951891249977052}
{"item_id":"senseval3.d000.s025.t006","gold_sense_keys":["offer%2:41:00::"],"candidates":[{"index":1,"sense_key":"offer%2:40:01::","synset_id":"offer.v.07"},{"index":2,"sense_key":"offer%2:41:00::","synset_id":"volunteer.v.02"},{"index":3,"sense_key":"offer%2:40:03::","synset_id":"offer.v.06"},{"index":4,"sense_key":"offer%2:40:09::","synset_id":"offer.v.05"},{"index":5,"sense_key":"offer%2:40:02::","synset_id":"offer.v.01"},{"index":6,"sense_key":"offer%2:31:00::","synset_id":"offer.v.04"},{"index":7,"sense_key":"offer%2:32:12::","synset_id":"propose.v.05"},{"index":8,"sense_key":"offer%2:42:00::","synset_id":"offer.v.09"},{"index":9,"sense_key":"offer%2:40:00::","synset_id":"offer.v.02"},{"index":10,"sense_key":"offer%2:41:03::","synset_id":"put_up.v.02"},{"index":11,"sense_key":"offer%2:40:10::","synset_id":"extend.v.04"},{"index":12,"sense_key":"offer%2:39:06::","synset_id":"offer.v.08"},{"index":13,"sense_key":"offer%2:32:06::","synset_id":"offer.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"offer%2:41:00::","call_ids":["senseval3.d000.s025.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"offer%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":810,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":126},"cost":{"currency":"USD","total_usd":0.008310000000000001,"input_uncached_usd":0.004050000000000001,"input_cached_usd":0.0,"output_usd":0.00426,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.190265250043012}
{"item_id":"senseval3.d000.s025.t007","gold_sense_keys":["buy%2:40:00::"],"candidates":[{"index":1,"sense_key":"buy%2:40:00::","synset_id":"buy.v.01"},{"index":2,"sense_key":"buy%2:40:03::","synset_id":"buy.v.04"},{"index":3,"sense_key":"buy%2:42:00::","synset_id":"buy.v.03"},{"index":4,"sense_key":"buy%2:31:09::","synset_id":"buy.v.05"},{"index":5,"sense_key":"buy%2:40:02::","synset_id":"bribe.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"buy%2:40:00::","call_ids":["senseval3.d000.s025.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"buy%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004275,"input_uncached_usd":0.002475,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3062233329983428}
{"item_id":"senseval3.d000.s025.t008","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":2,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":3,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":4,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":5,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s025.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.00575,"input_uncached_usd":0.00254,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9620006670011207}
{"item_id":"senseval3.d000.s026.t000","gold_sense_keys":["go%2:38:00::"],"candidates":[{"index":1,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":2,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":3,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"},{"index":4,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":5,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":6,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":7,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":8,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":9,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":10,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":11,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":12,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":13,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"},{"index":14,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":15,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":16,"sense_key":"go%2:30:08::","synset_id":"go.v.09"},{"index":17,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":18,"sense_key":"go%2:42:08::","synset_id":"go.v.25"},{"index":19,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":20,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":21,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":22,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":23,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":24,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"},{"index":25,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":26,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":27,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":28,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"},{"index":29,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":30,"sense_key":"go%2:38:01::","synset_id":"go.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":27,"chosen_sense_key":"go%2:38:00::","call_ids":["senseval3.d000.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":27,"predicted_sense_key":"go%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1691,"cached_input_tokens":0,"output_tokens":176,"reasoning_output_tokens":160},"cost":{"currency":"USD","total_usd":0.013735,"input_uncached_usd":0.008455,"input_cached_usd":0.0,"output_usd":0.00528,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.705267291981727}
{"item_id":"senseval3.d000.s026.t001","gold_sense_keys":["keep_to_oneself%2:41:00::"],"candidates":[{"index":1,"sense_key":"keep_to_oneself%2:41:00::","synset_id":"keep_to_oneself.v.01"},{"index":2,"sense_key":"keep_to_oneself%2:40:00::","synset_id":"keep_to_oneself.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"keep_to_oneself%2:41:00::","call_ids":["senseval3.d000.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"keep_to_oneself%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":372,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.0039000000000000003,"input_uncached_usd":0.00186,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2137880000518635}
{"item_id":"senseval3.d000.s026.t002","gold_sense_keys":["sit%2:35:00::","sit%2:38:00::"],"candidates":[{"index":1,"sense_key":"sit%2:41:05::","synset_id":"baby-sit.v.02"},{"index":2,"sense_key":"sit%2:38:03::","synset_id":"ride.v.01"},{"index":3,"sense_key":"sit%2:35:01::","synset_id":"seat.v.01"},{"index":4,"sense_key":"sit%2:38:00::","synset_id":"sit_down.v.01"},{"index":5,"sense_key":"sit%2:39:00::","synset_id":"model.v.03"},{"index":6,"sense_key":"sit%2:35:00::","synset_id":"sit.v.01"},{"index":7,"sense_key":"sit%2:38:01::","synset_id":"sit.v.04"},{"index":8,"sense_key":"sit%2:42:05::","synset_id":"sit.v.07"},{"index":9,"sense_key":"sit%2:42:00::","synset_id":"sit.v.02"},{"index":10,"sense_key":"sit%2:33:00::","synset_id":"sit.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"sit%2:35:00::","call_ids":["senseval3.d000.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"sit%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":403,"reasoning_output_tokens":387},"cost":{"currency":"USD","total_usd":0.01549,"input_uncached_usd":0.0034000000000000002,"input_cached_usd":0.0,"output_usd":0.01209,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.021261999965645}
{"item_id":"senseval3.d000.s026.t003","gold_sense_keys":["corner%1:06:02::"],"candidates":[{"index":1,"sense_key":"corner%1:25:00::","synset_id":"recess.n.02"},{"index":2,"sense_key":"corner%1:26:00::","synset_id":"corner.n.07"},{"index":3,"sense_key":"corner%1:06:02::","synset_id":"corner.n.03"},{"index":4,"sense_key":"corner%1:06:00::","synset_id":"corner.n.04"},{"index":5,"sense_key":"corner%1:17:00::","synset_id":"corner.n.09"},{"index":6,"sense_key":"corner%1:15:01::","synset_id":"corner.n.05"},{"index":7,"sense_key":"corner%1:26:01::","synset_id":"corner.n.08"},{"index":8,"sense_key":"corner%1:15:03::","synset_id":"corner.n.10"},{"index":9,"sense_key":"corner%1:15:00::","synset_id":"corner.n.02"},{"index":10,"sense_key":"corner%1:15:02::","synset_id":"corner.n.01"},{"index":11,"sense_key":"corner%1:06:01::","synset_id":"corner.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"corner%1:06:02::","call_ids":["senseval3.d000.s026.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"corner%1:06:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.00544,"input_uncached_usd":0.0034600000000000004,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7196188329253346}
{"item_id":"senseval3.d000.s026.t004","gold_sense_keys":["pass%2:42:00::"],"candidates":[{"index":1,"sense_key":"pass%2:38:01::","synset_id":"pass.v.07"},{"index":2,"sense_key":"pass%2:40:01::","synset_id":"pass.v.20"},{"index":3,"sense_key":"pass%2:38:00::","synset_id":"pass.v.01"},{"index":4,"sense_key":"pass%2:32:01::","synset_id":"communicate.v.01"},{"index":5,"sense_key":"pass%2:42:00::","synset_id":"spend.v.01"},{"index":6,"sense_key":"pass%2:41:04::","synset_id":"pass.v.09"},{"index":7,"sense_key":"pass%2:30:00::","synset_id":"happen.v.01"},{"index":8,"sense_key":"pass%2:30:01::","synset_id":"evanesce.v.01"},{"index":9,"sense_key":"pass%2:41:11::","synset_id":"pass.v.17"},{"index":10,"sense_key":"pass%2:41:08::","synset_id":"pass.v.14"},{"index":11,"sense_key":"pass%2:38:05::","synset_id":"travel_by.v.01"},{"index":12,"sense_key":"pass%2:29:00::","synset_id":"excrete.v.01"},{"index":13,"sense_key":"pass%2:38:03::","synset_id":"elapse.v.01"},{"index":14,"sense_key":"pass%2:35:00::","synset_id":"guide.v.05"},{"index":15,"sense_key":"pass%2:40:13::","synset_id":"fall.v.21"},{"index":16,"sense_key":"pass%2:42:04::","synset_id":"exceed.v.02"},{"index":17,"sense_key":"pass%2:41:02::","synset_id":"legislate.v.01"},{"index":18,"sense_key":"pass%2:38:04::","synset_id":"pass.v.22"},{"index":19,"sense_key":"pass%2:40:00::","synset_id":"pass.v.05"},{"index":20,"sense_key":"pass%2:30:11::","synset_id":"die.v.01"},{"index":21,"sense_key":"pass%2:40:02::","synset_id":"pass.v.18"},{"index":22,"sense_key":"pass%2:32:00::","synset_id":"authorize.v.01"},{"index":23,"sense_key":"pass%2:42:08::","synset_id":"run.v.03"},{"index":24,"sense_key":"pass%2:41:07::","synset_id":"pass.v.16"},{"index":25,"sense_key":"pass%2:38:02::","synset_id":"sink.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"pass%2:42:00::","call_ids":["senseval3.d000.s026.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"pass%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1432,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.00953,"input_uncached_usd":0.0071600000000000006,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4558113330276683}
{"item_id":"senseval3.d000.s026.t005","gold_sense_keys":["time%1:28:05::"],"candidates":[{"index":1,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":2,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"},{"index":5,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":8,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":9,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":10,"sense_key":"time%1:28:06::","synset_id":"time.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"time%1:28:00::","call_ids":["senseval3.d000.s026.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"time%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01933,"input_uncached_usd":0.0034900000000000005,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.761640458018519}
{"item_id":"senseval3.d000.s026.t006","gold_sense_keys":["nurse%2:37:00::"],"candidates":[{"index":1,"sense_key":"nurse%2:37:00::","synset_id":"harbor.v.01"},{"index":2,"sense_key":"nurse%2:29:00::","synset_id":"nurse.v.01"},{"index":3,"sense_key":"nurse%2:41:00::","synset_id":"nurse.v.03"},{"index":4,"sense_key":"nurse%2:34:00::","synset_id":"breastfeed.v.01"},{"index":5,"sense_key":"nurse%2:41:03::","synset_id":"nurse.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nurse%2:37:00::","call_ids":["senseval3.d000.s026.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nurse%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.00447,"input_uncached_usd":0.0026400000000000004,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.755900583928451}
{"item_id":"senseval3.d000.s026.t007","gold_sense_keys":["favorite%5:00:00:loved:00"],"candidates":[{"index":1,"sense_key":"favorite%5:00:00:loved:00","synset_id":"favored.s.01"},{"index":2,"sense_key":"favorite%5:00:00:popular:00","synset_id":"favorite.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"favorite%5:00:00:loved:00","call_ids":["senseval3.d000.s026.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"favorite%5:00:00:loved:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":380,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.00481,"input_uncached_usd":0.0019000000000000002,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.418565583997406}
{"item_id":"senseval3.d000.s027.t000","gold_sense_keys":["decide%2:31:00::"],"candidates":[{"index":1,"sense_key":"decide%2:31:02::","synset_id":"decide.v.03"},{"index":2,"sense_key":"decide%2:31:01::","synset_id":"decide.v.02"},{"index":3,"sense_key":"decide%2:31:00::","synset_id":"decide.v.01"},{"index":4,"sense_key":"decide%2:31:06::","synset_id":"decide.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"decide%2:31:00::","call_ids":["senseval3.d000.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"decide%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.003875,"input_uncached_usd":0.002285,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.51102020801045}
{"item_id":"senseval3.d000.s027.t001","gold_sense_keys":["mind%2:32:00::"],"candidates":[{"index":1,"sense_key":"mind%2:41:01::","synset_id":"take_care.v.02"},{"index":2,"sense_key":"mind%2:31:02::","synset_id":"mind.v.02"},{"index":3,"sense_key":"mind%2:31:00::","synset_id":"mind.v.06"},{"index":4,"sense_key":"mind%2:41:00::","synset_id":"heed.v.01"},{"index":5,"sense_key":"mind%2:31:03::","synset_id":"beware.v.01"},{"index":6,"sense_key":"mind%2:32:00::","synset_id":"mind.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"mind%2:32:00::","call_ids":["senseval3.d000.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"mind%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.004375,"input_uncached_usd":0.0025150000000000003,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0627239999594167}
{"item_id":"senseval3.d000.s027.t002","gold_sense_keys":["company%1:26:00::"],"candidates":[{"index":1,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":2,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":3,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":4,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":5,"sense_key":"company%1:14:05::","synset_id":"company.n.09"},{"index":6,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":7,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":8,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":9,"sense_key":"company%1:14:01::","synset_id":"company.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"company%1:26:00::","call_ids":["senseval3.d000.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"company%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.005265000000000001,"input_uncached_usd":0.0031950000000000004,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.909922541002743}
{"item_id":"senseval3.d000.s027.t004","gold_sense_keys":["free%5:00:00:unpaid:00"],"candidates":[{"index":1,"sense_key":"free%5:00:01:unoccupied:00","synset_id":"spare.s.03"},{"index":2,"sense_key":"free%5:00:00:nonexistent:00","synset_id":"barren.s.03"},{"index":3,"sense_key":"free%5:00:00:unpaid:00","synset_id":"complimentary.s.01"},{"index":4,"sense_key":"free%3:00:02::","synset_id":"free.a.02"},{"index":5,"sense_key":"free%3:00:00::","synset_id":"free.a.01"},{"index":6,"sense_key":"free%5:00:00:inexact:00","synset_id":"free.s.06"},{"index":7,"sense_key":"free%5:00:00:unfixed:00","synset_id":"detached.s.05"},{"index":8,"sense_key":"free%5:00:02:unoccupied:00","synset_id":"free.s.02"},{"index":9,"sense_key":"free%3:00:01::","synset_id":"free.a.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"free%5:00:00:unpaid:00","call_ids":["senseval3.d000.s027.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"free%5:00:00:unpaid:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":28},"cost":{"currency":"USD","total_usd":0.00472,"input_uncached_usd":0.0034000000000000002,"input_cached_usd":0.0,"output_usd":0.00132,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.58104533306323}
{"item_id":"senseval3.d000.s027.t005","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":2,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":3,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"},{"index":4,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":5,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s027.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.0046700000000000005,"input_uncached_usd":0.00251,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.317186542088166}
{"item_id":"senseval3.d000.s027.t006","gold_sense_keys":["make%2:40:01::"],"candidates":[{"index":1,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":2,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":3,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":4,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":5,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":6,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":9,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":10,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":11,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":12,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":13,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":14,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":15,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":16,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":19,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":20,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":21,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":22,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":23,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":24,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":25,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":26,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":27,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":28,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":29,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":30,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":31,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":32,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":33,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":34,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":35,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":36,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":37,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":38,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":39,"sense_key":"make%2:29:01::","synset_id":"make.v.49"},{"index":40,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":41,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":42,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":43,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":44,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":45,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":46,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":47,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":48,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":49,"sense_key":"make%2:36:05::","synset_id":"do.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":31,"chosen_sense_key":"make%2:40:01::","call_ids":["senseval3.d000.s027.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":31,"predicted_sense_key":"make%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2244,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.013170000000000001,"input_uncached_usd":0.01122,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6922343749320135}
{"item_id":"senseval3.d000.s027.t009","gold_sense_keys":["job%1:04:00::"],"candidates":[{"index":1,"sense_key":"job%1:06:00::","synset_id":"job.n.03"},{"index":2,"sense_key":"job%1:26:00::","synset_id":"problem.n.01"},{"index":3,"sense_key":"job%1:04:03::","synset_id":"job.n.07"},{"index":4,"sense_key":"job%1:04:05::","synset_id":"job.n.06"},{"index":5,"sense_key":"job%1:18:01::","synset_id":"job.n.10"},{"index":6,"sense_key":"job%1:06:01::","synset_id":"job.n.04"},{"index":7,"sense_key":"job%1:04:01::","synset_id":"job.n.02"},{"index":8,"sense_key":"job%1:04:00::","synset_id":"occupation.n.01"},{"index":9,"sense_key":"job%1:10:01::","synset_id":"job.n.11"},{"index":10,"sense_key":"job%1:04:02::","synset_id":"caper.n.03"},{"index":11,"sense_key":"job%1:04:04::","synset_id":"job.n.05"},{"index":12,"sense_key":"job%1:18:00::","synset_id":"job.n.09"},{"index":13,"sense_key":"job%1:10:00::","synset_id":"job.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"job%1:04:00::","call_ids":["senseval3.d000.s027.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"job%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":835,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0058850000000000005,"input_uncached_usd":0.004175000000000001,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4437289170455188}
{"item_id":"senseval3.d000.s029.t000","gold_sense_keys":["wonder%2:32:01::"],"candidates":[{"index":1,"sense_key":"wonder%2:32:12::","synset_id":"wonder.v.03"},{"index":2,"sense_key":"wonder%2:31:00::","synset_id":"wonder.v.01"},{"index":3,"sense_key":"wonder%2:32:01::","synset_id":"wonder.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"wonder%2:32:01::","call_ids":["senseval3.d000.s029.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"wonder%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":79},"cost":{"currency":"USD","total_usd":0.00487,"input_uncached_usd":0.00202,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8861641250550747}
{"item_id":"senseval3.d000.s029.t002","gold_sense_keys":["screwball%1:18:00::"],"candidates":[{"index":1,"sense_key":"screwball%1:18:00::","synset_id":"crackpot.n.01"},{"index":2,"sense_key":"screwball%1:04:00::","synset_id":"screwball.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"screwball%1:18:00::","call_ids":["senseval3.d000.s029.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"screwball%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":351,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.003225,"input_uncached_usd":0.001755,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2646660410100594}
{"item_id":"senseval3.d000.s030.t000","gold_sense_keys":["take%2:31:07::"],"candidates":[{"index":1,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":2,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":3,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":4,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":5,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":6,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":7,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":8,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":9,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":10,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":11,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":12,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":13,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":14,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":15,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":16,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":17,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":18,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":19,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":20,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":21,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":22,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":23,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":24,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":25,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":26,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":27,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":28,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":29,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":30,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":31,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":32,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":33,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":34,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":35,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":36,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":37,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"},{"index":38,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":39,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":40,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":41,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":42,"sense_key":"take%2:40:02::","synset_id":"take.v.33"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":28,"chosen_sense_key":"take%2:31:07::","call_ids":["senseval3.d000.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":28,"predicted_sense_key":"take%2:31:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2172,"cached_input_tokens":0,"output_tokens":215,"reasoning_output_tokens":199},"cost":{"currency":"USD","total_usd":0.017310000000000002,"input_uncached_usd":0.010860000000000002,"input_cached_usd":0.0,"output_usd":0.00645,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.633875250001438}
{"item_id":"senseval3.d000.s030.t001","gold_sense_keys":["offer%1:10:00::"],"candidates":[{"index":1,"sense_key":"offer%1:10:00::","synset_id":"offer.n.02"},{"index":2,"sense_key":"offer%1:10:01::","synset_id":"offer.n.01"},{"index":3,"sense_key":"offer%1:04:01::","synset_id":"crack.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"offer%1:10:00::","call_ids":["senseval3.d000.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"offer%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":263,"reasoning_output_tokens":248},"cost":{"currency":"USD","total_usd":0.010065000000000001,"input_uncached_usd":0.0021750000000000003,"input_cached_usd":0.0,"output_usd":0.00789,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.799193292041309}
{"item_id":"senseval3.d000.s030.t003","gold_sense_keys":["feel%2:29:00::","feel%2:31:01::","feel%2:37:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":3,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":4,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":5,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"},{"index":6,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":7,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":8,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":9,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":10,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":11,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":12,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":13,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"feel%2:37:00::","call_ids":["senseval3.d000.s030.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"feel%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":854,"cached_input_tokens":0,"output_tokens":466,"reasoning_output_tokens":450},"cost":{"currency":"USD","total_usd":0.01825,"input_uncached_usd":0.00427,"input_cached_usd":0.0,"output_usd":0.013980000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.304110915982164}
{"item_id":"senseval3.d000.s030.t004","gold_sense_keys":["uneasy%3:00:00::"],"candidates":[{"index":1,"sense_key":"uneasy%3:00:00::","synset_id":"uneasy.a.01"},{"index":2,"sense_key":"uneasy%5:00:00:uncomfortable:01","synset_id":"awkward.s.05"},{"index":3,"sense_key":"uneasy%3:00:05::","synset_id":"restless.a.03"},{"index":4,"sense_key":"uneasy%5:00:00:uncomfortable:00","synset_id":"uneasy.s.03"},{"index":5,"sense_key":"uneasy%5:00:00:troubled:00","synset_id":"anxious.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"uneasy%3:00:00::","call_ids":["senseval3.d000.s030.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"uneasy%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.034159999999999996,"input_uncached_usd":0.0027800000000000004,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":17.371779374894686}
{"item_id":"senseval3.d000.s031.t000","gold_sense_keys":["get%2:30:02::"],"candidates":[{"index":1,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":2,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":3,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":4,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":5,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":6,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":7,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":8,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":9,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":10,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":11,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"},{"index":12,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":13,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":14,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":15,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":16,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":17,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":18,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":19,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":20,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":21,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":22,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":23,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":24,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":25,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":26,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":27,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":28,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":29,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":30,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":31,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":32,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":33,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":34,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":35,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":36,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"get%2:30:02::","call_ids":["senseval3.d000.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"get%2:30:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1918,"cached_input_tokens":0,"output_tokens":174,"reasoning_output_tokens":159},"cost":{"currency":"USD","total_usd":0.01481,"input_uncached_usd":0.009590000000000001,"input_cached_usd":0.0,"output_usd":0.00522,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4542772920103744}
{"item_id":"senseval3.d000.s031.t003","gold_sense_keys":["give%2:40:07::"],"candidates":[{"index":1,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":2,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":3,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":4,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":5,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":6,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":7,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":8,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":9,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":10,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":11,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":12,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":13,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":14,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":15,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":16,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":17,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":18,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":19,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":20,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":21,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":22,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":23,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":24,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":25,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":26,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":27,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":28,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":29,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":30,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":31,"sense_key":"give%2:29:10::","synset_id":"give.v.44"},{"index":32,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":33,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":34,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":35,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":36,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":37,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":38,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":39,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":40,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":41,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":42,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":43,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":44,"sense_key":"give%2:29:11::","synset_id":"give.v.43"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"give%2:40:07::","call_ids":["senseval3.d000.s031.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"give%2:40:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2034,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.01293,"input_uncached_usd":0.01017,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9608885829802603}
{"item_id":"senseval3.d000.s031.t005","gold_sense_keys":["wait%2:42:00::"],"candidates":[{"index":1,"sense_key":"wait%2:41:00::","synset_id":"wait.v.04"},{"index":2,"sense_key":"wait%2:42:01::","synset_id":"wait.v.02"},{"index":3,"sense_key":"wait%2:42:00::","synset_id":"wait.v.01"},{"index":4,"sense_key":"wait%2:31:00::","synset_id":"expect.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"wait%2:42:00::","call_ids":["senseval3.d000.s031.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"wait%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":393,"reasoning_output_tokens":377},"cost":{"currency":"USD","total_usd":0.014095,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.01179,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.497848249971867}
{"item_id":"senseval3.d000.s031.t006","gold_sense_keys":["answer%1:10:01::"],"candidates":[{"index":1,"sense_key":"answer%1:10:00::","synset_id":"answer.n.03"},{"index":2,"sense_key":"answer%1:10:01::","synset_id":"answer.n.01"},{"index":3,"sense_key":"answer%1:04:00::","synset_id":"answer.n.05"},{"index":4,"sense_key":"answer%1:10:03::","synset_id":"answer.n.04"},{"index":5,"sense_key":"answer%1:10:02::","synset_id":"solution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"answer%1:10:01::","call_ids":["senseval3.d000.s031.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"answer%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":245,"reasoning_output_tokens":229},"cost":{"currency":"USD","total_usd":0.010165,"input_uncached_usd":0.0028150000000000002,"input_cached_usd":0.0,"output_usd":0.00735,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.557944832951762}
{"item_id":"senseval3.d000.s031.t007","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":2,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval3.d000.s031.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"seem%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":184,"reasoning_output_tokens":169},"cost":{"currency":"USD","total_usd":0.007865,"input_uncached_usd":0.0023450000000000003,"input_cached_usd":0.0,"output_usd":0.00552,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.749864542041905}
{"item_id":"senseval3.d000.s031.t008","gold_sense_keys":["wink%2:29:00::"],"candidates":[{"index":1,"sense_key":"wink%2:29:02::","synset_id":"wink.v.01"},{"index":2,"sense_key":"wink%2:29:01::","synset_id":"wink.v.04"},{"index":3,"sense_key":"wink%2:29:00::","synset_id":"blink.v.01"},{"index":4,"sense_key":"wink%2:39:00::","synset_id":"flash.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"wink%2:29:00::","call_ids":["senseval3.d000.s031.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"wink%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0040750000000000005,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8702620830154046}
{"item_id":"senseval3.d000.s031.t009","gold_sense_keys":["continue%2:42:01::"],"candidates":[{"index":1,"sense_key":"continue%2:30:10::","synset_id":"continue.v.10"},{"index":2,"sense_key":"continue%2:42:01::","synset_id":"continue.v.01"},{"index":3,"sense_key":"continue%2:42:12::","synset_id":"continue.v.06"},{"index":4,"sense_key":"continue%2:32:00::","synset_id":"continue.v.02"},{"index":5,"sense_key":"continue%2:38:00::","synset_id":"proceed.v.02"},{"index":6,"sense_key":"continue%2:42:06::","synset_id":"cover.v.03"},{"index":7,"sense_key":"continue%2:41:00::","synset_id":"retain.v.02"},{"index":8,"sense_key":"continue%2:42:11::","synset_id":"continue.v.07"},{"index":9,"sense_key":"continue%2:42:10::","synset_id":"stay.v.04"},{"index":10,"sense_key":"continue%2:42:00::","synset_id":"continue.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"continue%2:42:01::","call_ids":["senseval3.d000.s031.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"continue%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":776,"cached_input_tokens":0,"output_tokens":265,"reasoning_output_tokens":249},"cost":{"currency":"USD","total_usd":0.01183,"input_uncached_usd":0.00388,"input_cached_usd":0.0,"output_usd":0.00795,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.8261033340822905}
{"item_id":"senseval3.d000.s031.t010","gold_sense_keys":["stare%2:39:00::"],"candidates":[{"index":1,"sense_key":"stare%2:29:00::","synset_id":"stare.v.02"},{"index":2,"sense_key":"stare%2:39:00::","synset_id":"gaze.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stare%2:29:00::","call_ids":["senseval3.d000.s031.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stare%2:29:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":360,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01764,"input_uncached_usd":0.0018000000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.690649540978484}
{"item_id":"senseval3.d000.s032.t001","gold_sense_keys":["weak%5:00:00:perceptible:00"],"candidates":[{"index":1,"sense_key":"weak%5:00:00:powerless:00","synset_id":"weak.s.08"},{"index":2,"sense_key":"weak%5:00:00:regular:00","synset_id":"weak.s.07"},{"index":3,"sense_key":"weak%5:00:00:human:00","synset_id":"fallible.s.01"},{"index":4,"sense_key":"weak%5:00:00:unstressed:00","synset_id":"unaccented.s.01"},{"index":5,"sense_key":"weak%5:00:00:diluted:00","synset_id":"watery.s.04"},{"index":6,"sense_key":"weak%3:00:00::","synset_id":"weak.a.01"},{"index":7,"sense_key":"weak%5:00:00:perceptible:00","synset_id":"faint.s.01"},{"index":8,"sense_key":"weak%5:00:00:down:00","synset_id":"weak.s.04"},{"index":9,"sense_key":"weak%5:00:00:unskilled:00","synset_id":"weak.s.05"},{"index":10,"sense_key":"weak%5:00:00:fallible:00","synset_id":"weak.s.10"},{"index":11,"sense_key":"weak%5:00:00:stupid:00","synset_id":"weak.s.11"},{"index":12,"sense_key":"weak%5:00:00:frail:00","synset_id":"decrepit.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"weak%5:00:00:perceptible:00","call_ids":["senseval3.d000.s032.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"weak%5:00:00:perceptible:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":802,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.007220000000000001,"input_uncached_usd":0.0040100000000000005,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4753626249730587}
{"item_id":"senseval3.d000.s032.t002","gold_sense_keys":["laugh%1:10:00::"],"candidates":[{"index":1,"sense_key":"laugh%1:10:00::","synset_id":"laugh.n.01"},{"index":2,"sense_key":"laugh%1:10:02::","synset_id":"laugh.n.02"},{"index":3,"sense_key":"laugh%1:10:01::","synset_id":"joke.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"laugh%1:10:00::","call_ids":["senseval3.d000.s032.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"laugh%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.004185,"input_uncached_usd":0.001935,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9342656249646097}
{"item_id":"senseval3.d000.s034.t000","gold_sense_keys":["thanks%1:10:00::"],"candidates":[{"index":1,"sense_key":"thanks%1:04:00::","synset_id":"thanks.n.02"},{"index":2,"sense_key":"thanks%1:10:00::","synset_id":"thanks.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"thanks%1:10:00::","call_ids":["senseval3.d000.s034.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"thanks%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":312,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.0029100000000000003,"input_uncached_usd":0.0015600000000000002,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7692502080462873}
{"item_id":"senseval3.d000.s035.t000","gold_sense_keys":["faint%5:00:00:perceptible:00"],"candidates":[{"index":1,"sense_key":"faint%5:00:00:ill:01","synset_id":"faint.s.04"},{"index":2,"sense_key":"faint%5:00:00:perceptible:00","synset_id":"faint.s.01"},{"index":3,"sense_key":"faint%5:00:00:cowardly:00","synset_id":"faint.s.06"},{"index":4,"sense_key":"faint%5:00:00:weak:00","synset_id":"faint.s.03"},{"index":5,"sense_key":"faint%5:00:00:indistinct:00","synset_id":"dim.s.02"},{"index":6,"sense_key":"faint%5:00:02:indistinct:00","synset_id":"faint.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"faint%5:00:00:perceptible:00","call_ids":["senseval3.d000.s035.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"faint%5:00:00:perceptible:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":85},"cost":{"currency":"USD","total_usd":0.00576,"input_uncached_usd":0.0027300000000000002,"input_cached_usd":0.0,"output_usd":0.00303,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4641765829874203}
{"item_id":"senseval3.d000.s035.t001","gold_sense_keys":["crease%1:25:01::"],"candidates":[{"index":1,"sense_key":"crease%1:06:00::","synset_id":"kris.n.01"},{"index":2,"sense_key":"crease%1:25:01::","synset_id":"wrinkle.n.01"},{"index":3,"sense_key":"crease%1:25:02::","synset_id":"fold.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"crease%1:25:01::","call_ids":["senseval3.d000.s035.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"crease%1:25:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":393,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.003795,"input_uncached_usd":0.001965,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.831346874940209}
{"item_id":"senseval3.d000.s035.t002","gold_sense_keys":["appear%2:30:00::"],"candidates":[{"index":1,"sense_key":"appear%2:30:00::","synset_id":"appear.v.02"},{"index":2,"sense_key":"appear%2:36:00::","synset_id":"appear.v.06"},{"index":3,"sense_key":"appear%2:30:02::","synset_id":"appear.v.05"},{"index":4,"sense_key":"appear%2:41:03::","synset_id":"appear.v.07"},{"index":5,"sense_key":"appear%2:39:01::","synset_id":"appear.v.04"},{"index":6,"sense_key":"appear%2:30:01::","synset_id":"appear.v.03"},{"index":7,"sense_key":"appear%2:39:00::","synset_id":"look.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"appear%2:30:00::","call_ids":["senseval3.d000.s035.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"appear%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01882,"input_uncached_usd":0.0029800000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.039835708914325}
{"item_id":"senseval3.d000.s035.t003","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":2,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":3,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":4,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":5,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":6,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":7,"sense_key":"man%1:05:00::","synset_id":"world.n.08"},{"index":8,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":9,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":10,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":11,"sense_key":"man%1:06:00::","synset_id":"man.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s035.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":788,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.00628,"input_uncached_usd":0.00394,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.753022165969014}
{"item_id":"senseval3.d000.s036.t000","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":2,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":5,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":6,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":7,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":8,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":9,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":10,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":11,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":12,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"think%2:31:01::","call_ids":["senseval3.d000.s036.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"think%2:31:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":807,"cached_input_tokens":0,"output_tokens":182,"reasoning_output_tokens":166},"cost":{"currency":"USD","total_usd":0.009495,"input_uncached_usd":0.004035,"input_cached_usd":0.0,"output_usd":0.0054600000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.279726499924436}
{"item_id":"senseval3.d000.s036.t001","gold_sense_keys":["take%2:31:07::"],"candidates":[{"index":1,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":2,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":3,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":4,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":5,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":6,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":7,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":8,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":9,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":10,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":11,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":12,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":13,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":14,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":15,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":16,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":17,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":18,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":19,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":20,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":21,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":22,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":23,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":24,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":25,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":26,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":27,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":28,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":29,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":30,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":31,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":32,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":33,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":34,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":37,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":38,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":39,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":40,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"},{"index":41,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":42,"sense_key":"take%2:38:02::","synset_id":"take.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":25,"chosen_sense_key":"take%2:31:07::","call_ids":["senseval3.d000.s036.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":25,"predicted_sense_key":"take%2:31:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2102,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":122},"cost":{"currency":"USD","total_usd":0.014620000000000001,"input_uncached_usd":0.01051,"input_cached_usd":0.0,"output_usd":0.00411,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.6054561249911785}
{"item_id":"senseval3.d000.s037.t000","gold_sense_keys":["mean%2:32:01::"],"candidates":[{"index":1,"sense_key":"mean%2:32:03::","synset_id":"mean.v.03"},{"index":2,"sense_key":"mean%2:31:05::","synset_id":"mean.v.07"},{"index":3,"sense_key":"mean%2:42:03::","synset_id":"mean.v.05"},{"index":4,"sense_key":"mean%2:31:01::","synset_id":"think_of.v.04"},{"index":5,"sense_key":"mean%2:32:01::","synset_id":"mean.v.01"},{"index":6,"sense_key":"mean%2:42:00::","synset_id":"entail.v.01"},{"index":7,"sense_key":"mean%2:31:00::","synset_id":"intend.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"mean%2:32:01::","call_ids":["senseval3.d000.s037.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"mean%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":235,"reasoning_output_tokens":220},"cost":{"currency":"USD","total_usd":0.00962,"input_uncached_usd":0.0025700000000000002,"input_cached_usd":0.0,"output_usd":0.00705,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.932452083914541}
{"item_id":"senseval3.d000.s038.t000","gold_sense_keys":["have%2:40:00::"],"candidates":[{"index":1,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":2,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":3,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":4,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":5,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":6,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":7,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":8,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":9,"sense_key":"have%2:35:00::","synset_id":"take.v.35"},{"index":10,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":11,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":12,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":13,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":14,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":15,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":16,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":17,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":18,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":19,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d000.s038.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1044,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.007710000000000001,"input_uncached_usd":0.005220000000000001,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.934172832989134}
{"item_id":"senseval3.d000.s039.t000","gold_sense_keys":["hit%2:35:01::"],"candidates":[{"index":1,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"},{"index":2,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"},{"index":3,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":4,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":5,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"},{"index":6,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":7,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":8,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"},{"index":9,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":10,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":11,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":12,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":13,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":14,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"},{"index":15,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":16,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"},{"index":17,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"hit%2:35:01::","call_ids":["senseval3.d000.s039.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"hit%2:35:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":942,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":115},"cost":{"currency":"USD","total_usd":0.008640000000000002,"input_uncached_usd":0.004710000000000001,"input_cached_usd":0.0,"output_usd":0.00393,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.165362083935179}
{"item_id":"senseval3.d000.s039.t001","gold_sense_keys":["truck%1:06:00::"],"candidates":[{"index":1,"sense_key":"truck%1:06:00::","synset_id":"truck.n.01"},{"index":2,"sense_key":"truck%1:06:01::","synset_id":"hand_truck.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"truck%1:06:00::","call_ids":["senseval3.d000.s039.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"truck%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":285,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.002925,"input_uncached_usd":0.001425,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.912412124918774}
{"item_id":"senseval3.d000.s040.t000","gold_sense_keys":["return%2:40:03::"],"candidates":[{"index":1,"sense_key":"return%2:40:13::","synset_id":"fall.v.21"},{"index":2,"sense_key":"return%2:38:00::","synset_id":"return.v.01"},{"index":3,"sense_key":"return%2:41:00::","synset_id":"reelect.v.01"},{"index":4,"sense_key":"return%2:30:00::","synset_id":"revert.v.01"},{"index":5,"sense_key":"return%2:32:01::","synset_id":"retort.v.01"},{"index":6,"sense_key":"return%2:40:00::","synset_id":"refund.v.01"},{"index":7,"sense_key":"return%2:38:01::","synset_id":"return.v.14"},{"index":8,"sense_key":"return%2:32:00::","synset_id":"hark_back.v.01"},{"index":9,"sense_key":"return%2:35:01::","synset_id":"return.v.07"},{"index":10,"sense_key":"return%2:30:01::","synset_id":"come_back.v.01"},{"index":11,"sense_key":"return%2:38:12::","synset_id":"return.v.05"},{"index":12,"sense_key":"return%2:36:00::","synset_id":"render.v.04"},{"index":13,"sense_key":"return%2:40:08::","synset_id":"render.v.07"},{"index":14,"sense_key":"return%2:32:02::","synset_id":"return.v.16"},{"index":15,"sense_key":"return%2:32:03::","synset_id":"render.v.05"},{"index":16,"sense_key":"return%2:40:03::","synset_id":"return.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":16,"chosen_sense_key":"return%2:40:03::","call_ids":["senseval3.d000.s040.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":16,"predicted_sense_key":"return%2:40:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":806,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":116},"cost":{"currency":"USD","total_usd":0.007960000000000002,"input_uncached_usd":0.004030000000000001,"input_cached_usd":0.0,"output_usd":0.00393,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":135.4527765000239}
{"item_id":"senseval3.d000.s040.t001","gold_sense_keys":["favor%1:04:00::"],"candidates":[{"index":1,"sense_key":"favor%1:09:00::","synset_id":"favor.n.03"},{"index":2,"sense_key":"favor%1:06:00::","synset_id":"party_favor.n.01"},{"index":3,"sense_key":"favor%1:04:00::","synset_id":"favor.n.01"},{"index":4,"sense_key":"favor%1:07:00::","synset_id":"favor.n.02"},{"index":5,"sense_key":"favor%1:12:00::","synset_id":"favor.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"favor%1:04:00::","call_ids":["senseval3.d000.s040.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"favor%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":375,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.003585,"input_uncached_usd":0.0018750000000000001,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8722283750539646}
{"item_id":"senseval3.d000.s041.t000","gold_sense_keys":["do%2:36:01::","do%2:41:01::","do%2:41:04::"],"candidates":[{"index":1,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":2,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":3,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":4,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":5,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":8,"sense_key":"do%2:38:00::","synset_id":"do.v.13"},{"index":9,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":10,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":11,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":12,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":13,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"do%2:36:01::","call_ids":["senseval3.d000.s041.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":805,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":105},"cost":{"currency":"USD","total_usd":0.007655,"input_uncached_usd":0.004025,"input_cached_usd":0.0,"output_usd":0.00363,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.184531665989198}
{"item_id":"senseval3.d000.s041.t002","gold_sense_keys":["like%2:37:04::"],"candidates":[{"index":1,"sense_key":"like%2:31:00::","synset_id":"like.v.04"},{"index":2,"sense_key":"like%2:37:06::","synset_id":"like.v.05"},{"index":3,"sense_key":"like%2:37:05::","synset_id":"like.v.02"},{"index":4,"sense_key":"like%2:37:04::","synset_id":"like.v.03"},{"index":5,"sense_key":"like%2:37:01::","synset_id":"wish.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"like%2:37:04::","call_ids":["senseval3.d000.s041.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"like%2:37:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":386,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":90},"cost":{"currency":"USD","total_usd":0.00511,"input_uncached_usd":0.00193,"input_cached_usd":0.0,"output_usd":0.00318,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1947708330117166}
{"item_id":"senseval3.d000.s043.t001","gold_sense_keys":["try%2:37:00::","try%2:41:01::"],"candidates":[{"index":1,"sense_key":"try%2:37:01::","synset_id":"try.v.06"},{"index":2,"sense_key":"try%2:41:05::","synset_id":"hear.v.03"},{"index":3,"sense_key":"try%2:41:00::","synset_id":"try.v.01"},{"index":4,"sense_key":"try%2:41:06::","synset_id":"judge.v.05"},{"index":5,"sense_key":"try%2:30:00::","synset_id":"try.v.08"},{"index":6,"sense_key":"try%2:34:00::","synset_id":"sample.v.01"},{"index":7,"sense_key":"try%2:29:00::","synset_id":"try_on.v.01"},{"index":8,"sense_key":"try%2:37:00::","synset_id":"try.v.07"},{"index":9,"sense_key":"try%2:41:01::","synset_id":"test.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"try%2:41:01::","call_ids":["senseval3.d000.s043.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"try%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":202,"reasoning_output_tokens":186},"cost":{"currency":"USD","total_usd":0.009275,"input_uncached_usd":0.0032150000000000004,"input_cached_usd":0.0,"output_usd":0.00606,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8995504580670968}
{"item_id":"senseval3.d000.s044.t000","gold_sense_keys":["rub%2:35:00::"],"candidates":[{"index":1,"sense_key":"rub%2:39:00::","synset_id":"rub.v.03"},{"index":2,"sense_key":"rub%2:35:00::","synset_id":"rub.v.01"},{"index":3,"sense_key":"rub%2:35:01::","synset_id":"rub.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"rub%2:35:00::","call_ids":["senseval3.d000.s044.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"rub%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":318,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.0035700000000000003,"input_uncached_usd":0.00159,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.319910875055939}
{"item_id":"senseval3.d000.s044.t001","gold_sense_keys":["forehead%1:08:00::"],"candidates":[{"index":1,"sense_key":"forehead%1:08:00::","synset_id":"brow.n.01"},{"index":2,"sense_key":"forehead%1:08:01::","synset_id":"frontal_bone.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"forehead%1:08:00::","call_ids":["senseval3.d000.s044.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"forehead%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":288,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.0032700000000000003,"input_uncached_usd":0.00144,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8273569169687107}
{"item_id":"senseval3.d000.s045.t001","gold_sense_keys":["feel%2:29:00::","feel%2:31:01::","feel%2:37:00::"],"candidates":[{"index":1,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":2,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":3,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":4,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":5,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":6,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":7,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":8,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":9,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":10,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"},{"index":11,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":12,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":13,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"feel%2:29:00::","call_ids":["senseval3.d000.s045.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"feel%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":273,"reasoning_output_tokens":257},"cost":{"currency":"USD","total_usd":0.011955,"input_uncached_usd":0.003765,"input_cached_usd":0.0,"output_usd":0.00819,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.6608991250395775}
{"item_id":"senseval3.d000.s045.t002","gold_sense_keys":["woolly%5:00:00:confused:00"],"candidates":[{"index":1,"sense_key":"woolly%5:00:02:hairy:00","synset_id":"lanate.s.01"},{"index":2,"sense_key":"woolly%5:00:00:hairy:00","synset_id":"wooly.s.03"},{"index":3,"sense_key":"woolly%5:00:00:confused:00","synset_id":"addled.s.02"},{"index":4,"sense_key":"woolly%5:00:00:soft:01","synset_id":"flocculent.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"woolly%5:00:00:confused:00","call_ids":["senseval3.d000.s045.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"woolly%5:00:00:confused:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.003785,"input_uncached_usd":0.002075,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7674808340379968}
{"item_id":"senseval3.d000.s046.t001","gold_sense_keys":["guy%1:18:00::"],"candidates":[{"index":1,"sense_key":"guy%1:06:00::","synset_id":"guy.n.03"},{"index":2,"sense_key":"guy%1:06:01::","synset_id":"guy.n.02"},{"index":3,"sense_key":"guy%1:18:00::","synset_id":"guy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"guy%1:18:00::","call_ids":["senseval3.d000.s046.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"guy%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":352,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00353,"input_uncached_usd":0.00176,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8720105000538751}
{"item_id":"senseval3.d000.s046.t002","gold_sense_keys":["make%2:36:12::"],"candidates":[{"index":1,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":2,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":3,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":4,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":5,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":6,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":7,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":8,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":9,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":10,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":11,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":12,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":13,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":14,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":15,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":16,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":17,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":18,"sense_key":"make%2:29:01::","synset_id":"make.v.49"},{"index":19,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":20,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":21,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":22,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":23,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":24,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":25,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":26,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":27,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":28,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":29,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":30,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":31,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":32,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":33,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":34,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":35,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":36,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":37,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":38,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":39,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":40,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":41,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":42,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":43,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":44,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":45,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":46,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":47,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":48,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":49,"sense_key":"make%2:30:00::","synset_id":"make.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"make%2:36:12::","call_ids":["senseval3.d000.s046.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"make%2:36:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2176,"cached_input_tokens":0,"output_tokens":175,"reasoning_output_tokens":159},"cost":{"currency":"USD","total_usd":0.016130000000000002,"input_uncached_usd":0.01088,"input_cached_usd":0.0,"output_usd":0.00525,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1651086249621585}
{"item_id":"senseval3.d000.s046.t003","gold_sense_keys":["exit%1:04:00::"],"candidates":[{"index":1,"sense_key":"exit%1:11:00::","synset_id":"passing.n.02"},{"index":2,"sense_key":"exit%1:06:00::","synset_id":"exit.n.01"},{"index":3,"sense_key":"exit%1:04:00::","synset_id":"exit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"exit%1:04:00::","call_ids":["senseval3.d000.s046.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"exit%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":334,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.00326,"input_uncached_usd":0.00167,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.863102708011866}
{"item_id":"senseval3.d000.s047.t000","gold_sense_keys":["expect%2:31:00::"],"candidates":[{"index":1,"sense_key":"expect%2:31:05::","synset_id":"expect.v.04"},{"index":2,"sense_key":"expect%2:29:00::","synset_id":"have_a_bun_in_the_oven.v.01"},{"index":3,"sense_key":"expect%2:31:00::","synset_id":"expect.v.01"},{"index":4,"sense_key":"expect%2:32:00::","synset_id":"ask.v.04"},{"index":5,"sense_key":"expect%2:31:01::","synset_id":"expect.v.03"},{"index":6,"sense_key":"expect%2:29:02::","synset_id":"expect.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"expect%2:31:00::","call_ids":["senseval3.d000.s047.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"expect%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.0052250000000000005,"input_uncached_usd":0.002465,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.996064000064507}
{"item_id":"senseval3.d000.s047.t001","gold_sense_keys":["come%2:30:13::"],"candidates":[{"index":1,"sense_key":"come%2:31:13::","synset_id":"come.v.21"},{"index":2,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":3,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":4,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":5,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":6,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":7,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":8,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":9,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":10,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":11,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":12,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":13,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":14,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":15,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":16,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":19,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":20,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":21,"sense_key":"come%2:38:00::","synset_id":"come.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"come%2:30:13::","call_ids":["senseval3.d000.s047.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"come%2:30:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1105,"cached_input_tokens":0,"output_tokens":388,"reasoning_output_tokens":372},"cost":{"currency":"USD","total_usd":0.017165,"input_uncached_usd":0.005525,"input_cached_usd":0.0,"output_usd":0.011640000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.008240166003816}
{"item_id":"senseval3.d000.s047.t003","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval3.d000.s047.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":85},"cost":{"currency":"USD","total_usd":0.00508,"input_uncached_usd":0.00205,"input_cached_usd":0.0,"output_usd":0.00303,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4150725409854203}
{"item_id":"senseval3.d000.s048.t000","gold_sense_keys":["concede%2:32:01::"],"candidates":[{"index":1,"sense_key":"concede%2:32:00::","synset_id":"concede.v.01"},{"index":2,"sense_key":"concede%2:32:01::","synset_id":"concede.v.02"},{"index":3,"sense_key":"concede%2:40:00::","synset_id":"concede.v.03"},{"index":4,"sense_key":"concede%2:33:00::","synset_id":"concede.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"concede%2:32:01::","call_ids":["senseval3.d000.s048.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"concede%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":90},"cost":{"currency":"USD","total_usd":0.00517,"input_uncached_usd":0.00202,"input_cached_usd":0.0,"output_usd":0.00315,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9191565420478582}
{"item_id":"senseval3.d000.s048.t001","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":3,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":4,"sense_key":"know%2:31:14::","synset_id":"know.v.11"},{"index":5,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":6,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":7,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":8,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":11,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval3.d000.s048.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":757,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":129},"cost":{"currency":"USD","total_usd":0.008135,"input_uncached_usd":0.003785,"input_cached_usd":0.0,"output_usd":0.00435,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6347236250294372}
{"item_id":"senseval3.d000.s049.t000","gold_sense_keys":["keep%2:42:07::"],"candidates":[{"index":1,"sense_key":"keep%2:42:12::","synset_id":"keep.v.16"},{"index":2,"sense_key":"keep%2:35:10::","synset_id":"keep.v.21"},{"index":3,"sense_key":"keep%2:40:00::","synset_id":"keep.v.03"},{"index":4,"sense_key":"keep%2:34:00::","synset_id":"sustain.v.04"},{"index":5,"sense_key":"keep%2:41:02::","synset_id":"retain.v.02"},{"index":6,"sense_key":"keep%2:41:05::","synset_id":"observe.v.06"},{"index":7,"sense_key":"keep%2:40:12::","synset_id":"keep.v.19"},{"index":8,"sense_key":"keep%2:40:01::","synset_id":"keep.v.07"},{"index":9,"sense_key":"keep%2:31:00::","synset_id":"observe.v.08"},{"index":10,"sense_key":"keep%2:42:00::","synset_id":"keep.v.01"},{"index":11,"sense_key":"keep%2:41:00::","synset_id":"restrain.v.01"},{"index":12,"sense_key":"keep%2:41:03::","synset_id":"observe.v.09"},{"index":13,"sense_key":"keep%2:42:01::","synset_id":"keep.v.09"},{"index":14,"sense_key":"keep%2:41:01::","synset_id":"prevent.v.02"},{"index":15,"sense_key":"keep%2:40:10::","synset_id":"keep.v.20"},{"index":16,"sense_key":"keep%2:30:10::","synset_id":"preserve.v.04"},{"index":17,"sense_key":"keep%2:42:03::","synset_id":"keep.v.12"},{"index":18,"sense_key":"keep%2:42:02::","synset_id":"keep.v.15"},{"index":19,"sense_key":"keep%2:40:09::","synset_id":"keep_open.v.01"},{"index":20,"sense_key":"keep%2:40:13::","synset_id":"keep.v.18"},{"index":21,"sense_key":"keep%2:42:07::","synset_id":"continue.v.01"},{"index":22,"sense_key":"keep%2:32:00::","synset_id":"keep.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":21,"chosen_sense_key":"keep%2:42:07::","call_ids":["senseval3.d000.s049.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":21,"predicted_sense_key":"keep%2:42:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1180,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":103},"cost":{"currency":"USD","total_usd":0.009470000000000001,"input_uncached_usd":0.005900000000000001,"input_cached_usd":0.0,"output_usd":0.0035700000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.605204499908723}
{"item_id":"senseval3.d000.s049.t001","gold_sense_keys":["ride%2:32:00::"],"candidates":[{"index":1,"sense_key":"ride%2:38:01::","synset_id":"ride.v.01"},{"index":2,"sense_key":"ride%2:42:07::","synset_id":"depend_on.v.01"},{"index":3,"sense_key":"ride%2:35:00::","synset_id":"ride.v.13"},{"index":4,"sense_key":"ride%2:38:04::","synset_id":"ride.v.10"},{"index":5,"sense_key":"ride%2:42:00::","synset_id":"ride.v.03"},{"index":6,"sense_key":"ride%2:32:00::","synset_id":"tease.v.02"},{"index":7,"sense_key":"ride%2:38:00::","synset_id":"ride.v.02"},{"index":8,"sense_key":"ride%2:42:04::","synset_id":"ride.v.06"},{"index":9,"sense_key":"ride%2:38:03::","synset_id":"ride.v.11"},{"index":10,"sense_key":"ride%2:42:02::","synset_id":"ride.v.09"},{"index":11,"sense_key":"ride%2:38:02::","synset_id":"ride.v.04"},{"index":12,"sense_key":"ride%2:42:03::","synset_id":"drive.v.12"},{"index":13,"sense_key":"ride%2:35:03::","synset_id":"ride.v.14"},{"index":14,"sense_key":"ride%2:38:07::","synset_id":"ride.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"ride%2:32:00::","call_ids":["senseval3.d000.s049.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"ride%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":822,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.00573,"input_uncached_usd":0.00411,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6902703339001164}
{"item_id":"senseval3.d000.s049.t002","gold_sense_keys":["like%2:37:05::"],"candidates":[{"index":1,"sense_key":"like%2:37:06::","synset_id":"like.v.05"},{"index":2,"sense_key":"like%2:37:04::","synset_id":"like.v.03"},{"index":3,"sense_key":"like%2:37:01::","synset_id":"wish.v.02"},{"index":4,"sense_key":"like%2:31:00::","synset_id":"like.v.04"},{"index":5,"sense_key":"like%2:37:05::","synset_id":"like.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"like%2:37:05::","call_ids":["senseval3.d000.s049.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"like%2:37:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":109},"cost":{"currency":"USD","total_usd":0.005885,"input_uncached_usd":0.002135,"input_cached_usd":0.0,"output_usd":0.0037500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4391742919106036}
{"item_id":"senseval3.d000.s049.t003","gold_sense_keys":["listen%2:39:00::"],"candidates":[{"index":1,"sense_key":"listen%2:39:00::","synset_id":"listen.v.01"},{"index":2,"sense_key":"listen%2:39:01::","synset_id":"listen.v.02"},{"index":3,"sense_key":"listen%2:41:11::","synset_id":"heed.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"listen%2:39:00::","call_ids":["senseval3.d000.s049.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"listen%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":366,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.0036,"input_uncached_usd":0.0018300000000000002,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1175902080722153}
{"item_id":"senseval3.d000.s049.t004","gold_sense_keys":["radio%1:06:01::"],"candidates":[{"index":1,"sense_key":"radio%1:06:00::","synset_id":"radio.n.03"},{"index":2,"sense_key":"radio%1:06:01::","synset_id":"radio_receiver.n.01"},{"index":3,"sense_key":"radio%1:10:00::","synset_id":"radio.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"radio%1:06:01::","call_ids":["senseval3.d000.s049.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"radio%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":352,"cached_input_tokens":0,"output_tokens":991,"reasoning_output_tokens":969},"cost":{"currency":"USD","total_usd":0.031490000000000004,"input_uncached_usd":0.00176,"input_cached_usd":0.0,"output_usd":0.02973,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":15.284249125048518}
{"item_id":"senseval3.d000.s049.t005","gold_sense_keys":["sing%2:36:00::"],"candidates":[{"index":1,"sense_key":"sing%2:32:01::","synset_id":"spill_the_beans.v.01"},{"index":2,"sense_key":"sing%2:32:00::","synset_id":"whistle.v.05"},{"index":3,"sense_key":"sing%2:32:08::","synset_id":"sing.v.03"},{"index":4,"sense_key":"sing%2:36:01::","synset_id":"sing.v.01"},{"index":5,"sense_key":"sing%2:36:00::","synset_id":"sing.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"sing%2:36:00::","call_ids":["senseval3.d000.s049.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"sing%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":106},"cost":{"currency":"USD","total_usd":0.005895,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.00366,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7390317081008106}
{"item_id":"senseval3.d000.s049.t007","gold_sense_keys":["bath%1:04:00::"],"candidates":[{"index":1,"sense_key":"bath%1:06:01::","synset_id":"bathroom.n.01"},{"index":2,"sense_key":"bath%1:23:00::","synset_id":"bath.n.04"},{"index":3,"sense_key":"bath%1:04:00::","synset_id":"bath.n.02"},{"index":4,"sense_key":"bath%1:06:00::","synset_id":"bathtub.n.01"},{"index":5,"sense_key":"bath%1:15:00::","synset_id":"bath.n.05"},{"index":6,"sense_key":"bath%1:06:02::","synset_id":"bath.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bath%1:04:00::","call_ids":["senseval3.d000.s049.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bath%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.004105,"input_uncached_usd":0.002425,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6565620420733467}
{"item_id":"senseval3.d000.s050.t000","gold_sense_keys":["say%2:32:01::","say%2:32:13::"],"candidates":[{"index":1,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":2,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":3,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":4,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":5,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":6,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":9,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":10,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":11,"sense_key":"say%2:32:15::","synset_id":"say.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval3.d000.s050.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":736,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":120},"cost":{"currency":"USD","total_usd":0.00776,"input_uncached_usd":0.00368,"input_cached_usd":0.0,"output_usd":0.00408,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.683373667066917}
{"item_id":"senseval3.d000.s050.t001","gold_sense_keys":["neighbor%1:18:00::"],"candidates":[{"index":1,"sense_key":"neighbor%1:17:00::","synset_id":"neighbor.n.02"},{"index":2,"sense_key":"neighbor%1:18:00::","synset_id":"neighbor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"neighbor%1:18:00::","call_ids":["senseval3.d000.s050.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"neighbor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":332,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.00322,"input_uncached_usd":0.0016600000000000002,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8426585830748081}
{"item_id":"senseval3.d000.s050.t002","gold_sense_keys":["complain%2:32:00::"],"candidates":[{"index":1,"sense_key":"complain%2:32:00::","synset_id":"complain.v.01"},{"index":2,"sense_key":"complain%2:32:01::","synset_id":"complain.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"complain%2:32:00::","call_ids":["senseval3.d000.s050.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"complain%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":357,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.003135,"input_uncached_usd":0.0017850000000000001,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6460890830494463}
{"item_id":"senseval3.d000.s050.t003","gold_sense_keys":["believe%2:31:00::"],"candidates":[{"index":1,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"},{"index":2,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":3,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"},{"index":4,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"},{"index":5,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"believe%2:31:00::","call_ids":["senseval3.d000.s050.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"believe%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.005335,"input_uncached_usd":0.0023650000000000003,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6762542909709737}
{"item_id":"senseval3.d000.s051.t000","gold_sense_keys":["tell%2:32:00::"],"candidates":[{"index":1,"sense_key":"tell%2:32:08::","synset_id":"assure.v.02"},{"index":2,"sense_key":"tell%2:32:00::","synset_id":"tell.v.02"},{"index":3,"sense_key":"tell%2:31:01::","synset_id":"distinguish.v.01"},{"index":4,"sense_key":"tell%2:31:00::","synset_id":"tell.v.05"},{"index":5,"sense_key":"tell%2:32:03::","synset_id":"tell.v.07"},{"index":6,"sense_key":"tell%2:32:04::","synset_id":"state.v.01"},{"index":7,"sense_key":"tell%2:32:02::","synset_id":"order.v.01"},{"index":8,"sense_key":"tell%2:32:01::","synset_id":"tell.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tell%2:32:00::","call_ids":["senseval3.d000.s051.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tell%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":645,"reasoning_output_tokens":623},"cost":{"currency":"USD","total_usd":0.02228,"input_uncached_usd":0.0029300000000000003,"input_cached_usd":0.0,"output_usd":0.01935,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.866533959051594}
{"item_id":"senseval3.d000.s051.t001","gold_sense_keys":["bother%2:37:00::"],"candidates":[{"index":1,"sense_key":"bother%2:37:02::","synset_id":"bother.v.06"},{"index":2,"sense_key":"bother%2:41:01::","synset_id":"trouble.v.02"},{"index":3,"sense_key":"bother%2:37:00::","synset_id":"annoy.v.01"},{"index":4,"sense_key":"bother%2:41:00::","synset_id":"trouble_oneself.v.01"},{"index":5,"sense_key":"bother%2:38:00::","synset_id":"bother.v.04"},{"index":6,"sense_key":"bother%2:37:03::","synset_id":"bother.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bother%2:37:00::","call_ids":["senseval3.d000.s051.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bother%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":79},"cost":{"currency":"USD","total_usd":0.005520000000000001,"input_uncached_usd":0.00267,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.293537374935113}
{"item_id":"senseval3.d000.s052.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":2,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":3,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":4,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":5,"sense_key":"man%1:05:00::","synset_id":"world.n.08"},{"index":6,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":7,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":11,"sense_key":"man%1:06:00::","synset_id":"man.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s052.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":786,"cached_input_tokens":0,"output_tokens":178,"reasoning_output_tokens":162},"cost":{"currency":"USD","total_usd":0.00927,"input_uncached_usd":0.00393,"input_cached_usd":0.0,"output_usd":0.00534,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1964164579985663}
{"item_id":"senseval3.d000.s052.t001","gold_sense_keys":["close%2:35:00::"],"candidates":[{"index":1,"sense_key":"close%2:38:10::","synset_id":"close.v.11"},{"index":2,"sense_key":"close%2:41:00::","synset_id":"close_up.v.01"},{"index":3,"sense_key":"close%2:41:15::","synset_id":"close.v.08"},{"index":4,"sense_key":"close%2:38:08::","synset_id":"close.v.10"},{"index":5,"sense_key":"close%2:32:05::","synset_id":"close.v.06"},{"index":6,"sense_key":"close%2:35:07::","synset_id":"close.v.14"},{"index":7,"sense_key":"close%2:41:01::","synset_id":"close.v.04"},{"index":8,"sense_key":"close%2:42:01::","synset_id":"close.v.07"},{"index":9,"sense_key":"close%2:38:15::","synset_id":"close.v.12"},{"index":10,"sense_key":"close%2:35:15::","synset_id":"close_up.v.03"},{"index":11,"sense_key":"close%2:35:11::","synset_id":"close.v.15"},{"index":12,"sense_key":"close%2:30:08::","synset_id":"close.v.17"},{"index":13,"sense_key":"close%2:39:10::","synset_id":"close.v.09"},{"index":14,"sense_key":"close%2:35:00::","synset_id":"close.v.01"},{"index":15,"sense_key":"close%2:35:06::","synset_id":"close.v.02"},{"index":16,"sense_key":"close%2:42:00::","synset_id":"conclude.v.04"},{"index":17,"sense_key":"close%2:35:14::","synset_id":"close.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"close%2:35:00::","call_ids":["senseval3.d000.s052.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"close%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":918,"cached_input_tokens":0,"output_tokens":169,"reasoning_output_tokens":153},"cost":{"currency":"USD","total_usd":0.009660000000000002,"input_uncached_usd":0.00459,"input_cached_usd":0.0,"output_usd":0.00507,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.904611959005706}
{"item_id":"senseval3.d000.s052.t002","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s052.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.003940000000000001,"input_uncached_usd":0.00223,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9448087910423055}
{"item_id":"senseval3.d000.s052.t003","gold_sense_keys":["nod%2:32:00::"],"candidates":[{"index":1,"sense_key":"nod%2:29:02::","synset_id":"nod.v.05"},{"index":2,"sense_key":"nod%2:29:03::","synset_id":"nod.v.03"},{"index":3,"sense_key":"nod%2:38:00::","synset_id":"nod.v.04"},{"index":4,"sense_key":"nod%2:32:01::","synset_id":"nod.v.01"},{"index":5,"sense_key":"nod%2:32:00::","synset_id":"nod.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"nod%2:32:00::","call_ids":["senseval3.d000.s052.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"nod%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":248,"reasoning_output_tokens":232},"cost":{"currency":"USD","total_usd":0.00961,"input_uncached_usd":0.00217,"input_cached_usd":0.0,"output_usd":0.00744,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.314221084001474}
{"item_id":"senseval3.d000.s053.t000","gold_sense_keys":["look%2:39:00::"],"candidates":[{"index":1,"sense_key":"look%2:31:02::","synset_id":"count.v.08"},{"index":2,"sense_key":"look%2:29:00::","synset_id":"look.v.03"},{"index":3,"sense_key":"look%2:42:00::","synset_id":"front.v.01"},{"index":4,"sense_key":"look%2:39:02::","synset_id":"search.v.02"},{"index":5,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":6,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":7,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":8,"sense_key":"look%2:39:00::","synset_id":"look.v.01"},{"index":9,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":10,"sense_key":"look%2:39:01::","synset_id":"look.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"look%2:39:00::","call_ids":["senseval3.d000.s053.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"look%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":173,"reasoning_output_tokens":157},"cost":{"currency":"USD","total_usd":0.00859,"input_uncached_usd":0.0034000000000000002,"input_cached_usd":0.0,"output_usd":0.00519,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4363901669858024}
{"item_id":"senseval3.d000.s053.t001","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":2,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":3,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval3.d000.s053.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"seem%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.00558,"input_uncached_usd":0.0021000000000000003,"input_cached_usd":0.0,"output_usd":0.00348,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.774216250050813}
{"item_id":"senseval3.d000.s054.t000","gold_sense_keys":["fine%4:02:00::"],"candidates":[{"index":1,"sense_key":"fine%4:02:00::","synset_id":"very_well.r.02"},{"index":2,"sense_key":"fine%4:02:02::","synset_id":"finely.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fine%4:02:00::","call_ids":["senseval3.d000.s054.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fine%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":302,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.00298,"input_uncached_usd":0.00151,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5064642080105841}
{"item_id":"senseval3.d000.s055.t000","gold_sense_keys":["give%2:40:07::"],"candidates":[{"index":1,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":2,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":3,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":4,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":5,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":6,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":7,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":8,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":9,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":10,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":11,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":14,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":15,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":16,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":17,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":18,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":19,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":20,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":21,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":22,"sense_key":"give%2:29:10::","synset_id":"give.v.44"},{"index":23,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":24,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":25,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":26,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":27,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":28,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":29,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":30,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":31,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":32,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":33,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":34,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":35,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":36,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":37,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":38,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":39,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":40,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":41,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":42,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":43,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":44,"sense_key":"give%2:36:03::","synset_id":"give.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"give%2:40:07::","call_ids":["senseval3.d000.s055.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"give%2:40:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1955,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":698},"cost":{"currency":"USD","total_usd":0.031375,"input_uncached_usd":0.009775,"input_cached_usd":0.0,"output_usd":0.0216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.711737624951638}
{"item_id":"senseval3.d000.s055.t001","gold_sense_keys":["address%1:15:00::"],"candidates":[{"index":1,"sense_key":"address%1:10:01::","synset_id":"address.n.05"},{"index":2,"sense_key":"address%1:15:00::","synset_id":"address.n.02"},{"index":3,"sense_key":"address%1:10:03::","synset_id":"address.n.04"},{"index":4,"sense_key":"address%1:07:00::","synset_id":"savoir-faire.n.01"},{"index":5,"sense_key":"address%1:10:04::","synset_id":"address.n.01"},{"index":6,"sense_key":"address%1:10:00::","synset_id":"address.n.03"},{"index":7,"sense_key":"address%1:07:01::","synset_id":"address.n.07"},{"index":8,"sense_key":"address%1:10:02::","synset_id":"address.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"address%1:15:00::","call_ids":["senseval3.d000.s055.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"address%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01828,"input_uncached_usd":0.0024400000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.820766708930023}
{"item_id":"senseval3.d000.s056.t000","gold_sense_keys":["take%2:40:06::"],"candidates":[{"index":1,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":2,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":3,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":4,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":5,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":6,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":7,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":8,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":9,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":10,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"},{"index":11,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":12,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":13,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":14,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":15,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":16,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":17,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":18,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":19,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":20,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":21,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":22,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":23,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":24,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":25,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":26,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":27,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":28,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":29,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":30,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":31,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":32,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":33,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":34,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":35,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":36,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":37,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":38,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":39,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":40,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":40,"chosen_sense_key":"take%2:40:06::","call_ids":["senseval3.d000.s056.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":40,"predicted_sense_key":"take%2:40:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2069,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.011695,"input_uncached_usd":0.010345,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.054624375072308}
{"item_id":"senseval3.d000.s056.t001","gold_sense_keys":["time%1:03:00::","time%1:28:05::"],"candidates":[{"index":1,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":2,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":3,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"},{"index":4,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":7,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":8,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":9,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":10,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"time%1:28:05::","call_ids":["senseval3.d000.s056.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"time%1:28:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":193,"reasoning_output_tokens":177},"cost":{"currency":"USD","total_usd":0.00884,"input_uncached_usd":0.00305,"input_cached_usd":0.0,"output_usd":0.00579,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.738561708014458}
{"item_id":"senseval3.d000.s057.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":2,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":3,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":4,"sense_key":"want%2:42:01::","synset_id":"want.v.05"},{"index":5,"sense_key":"want%2:34:00::","synset_id":"want.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"want%2:37:00::","call_ids":["senseval3.d000.s057.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":99},"cost":{"currency":"USD","total_usd":0.00547,"input_uncached_usd":0.00202,"input_cached_usd":0.0,"output_usd":0.00345,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":141.15668587491382}
{"item_id":"senseval3.d000.s057.t001","gold_sense_keys":["study%2:31:02::"],"candidates":[{"index":1,"sense_key":"study%2:31:01::","synset_id":"study.v.05"},{"index":2,"sense_key":"study%2:31:02::","synset_id":"analyze.v.01"},{"index":3,"sense_key":"study%2:31:04::","synset_id":"study.v.06"},{"index":4,"sense_key":"study%2:31:03::","synset_id":"study.v.02"},{"index":5,"sense_key":"study%2:31:00::","synset_id":"learn.v.04"},{"index":6,"sense_key":"study%2:39:00::","synset_id":"study.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"study%2:31:02::","call_ids":["senseval3.d000.s057.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"study%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":98},"cost":{"currency":"USD","total_usd":0.005805,"input_uncached_usd":0.0023850000000000004,"input_cached_usd":0.0,"output_usd":0.0034200000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.345083541003987}
{"item_id":"senseval3.d000.s057.t003","gold_sense_keys":["habit%1:09:00::"],"candidates":[{"index":1,"sense_key":"habit%1:06:01::","synset_id":"habit.n.05"},{"index":2,"sense_key":"habit%1:04:01::","synset_id":"substance_abuse.n.01"},{"index":3,"sense_key":"habit%1:04:02::","synset_id":"habit.n.02"},{"index":4,"sense_key":"habit%1:06:00::","synset_id":"habit.n.03"},{"index":5,"sense_key":"habit%1:22:00::","synset_id":"habit.n.04"},{"index":6,"sense_key":"habit%1:09:00::","synset_id":"habit.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"habit%1:09:00::","call_ids":["senseval3.d000.s057.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"habit%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01807,"input_uncached_usd":0.00223,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.315367790986784}
{"item_id":"senseval3.d000.s057.t004","gold_sense_keys":["movement%1:04:04::"],"candidates":[{"index":1,"sense_key":"movement%1:04:03::","synset_id":"campaign.n.02"},{"index":2,"sense_key":"movement%1:04:02::","synset_id":"movement.n.11"},{"index":3,"sense_key":"movement%1:10:00::","synset_id":"movement.n.05"},{"index":4,"sense_key":"movement%1:06:00::","synset_id":"movement.n.10"},{"index":5,"sense_key":"movement%1:04:04::","synset_id":"motion.n.06"},{"index":6,"sense_key":"movement%1:11:00::","synset_id":"movement.n.03"},{"index":7,"sense_key":"movement%1:14:00::","synset_id":"movement.n.04"},{"index":8,"sense_key":"movement%1:09:00::","synset_id":"drift.n.05"},{"index":9,"sense_key":"movement%1:22:00::","synset_id":"bowel_movement.n.01"},{"index":10,"sense_key":"movement%1:19:00::","synset_id":"apparent_motion.n.01"},{"index":11,"sense_key":"movement%1:04:00::","synset_id":"motion.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"movement%1:04:04::","call_ids":["senseval3.d000.s057.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"movement%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":751,"cached_input_tokens":0,"output_tokens":165,"reasoning_output_tokens":150},"cost":{"currency":"USD","total_usd":0.008705000000000001,"input_uncached_usd":0.003755,"input_cached_usd":0.0,"output_usd":0.00495,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7389816669747233}
{"item_id":"senseval3.d000.s058.t000","gold_sense_keys":["see%2:31:01::"],"candidates":[{"index":1,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":2,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":3,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":4,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":5,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":6,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":7,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":8,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"},{"index":9,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":10,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":11,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":12,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":13,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":16,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":17,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":18,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":19,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":20,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":21,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":22,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":23,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":24,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"see%2:31:01::","call_ids":["senseval3.d000.s058.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"see%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1350,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":73},"cost":{"currency":"USD","total_usd":0.009420000000000001,"input_uncached_usd":0.006750000000000001,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6927887500496581}
{"item_id":"senseval3.d000.s058.t001","gold_sense_keys":["make%2:30:00::"],"candidates":[{"index":1,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":2,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":3,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":4,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":5,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":6,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":7,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":8,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":9,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":10,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":11,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":12,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":13,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":15,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":16,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":17,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":18,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":19,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":20,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":21,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":22,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":23,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":24,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":25,"sense_key":"make%2:29:01::","synset_id":"make.v.49"},{"index":26,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":27,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":28,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":29,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":30,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":31,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":32,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":33,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":34,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":35,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":36,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":37,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":38,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":39,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":40,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":41,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":42,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":43,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":44,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":45,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":46,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":47,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":48,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":49,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":26,"chosen_sense_key":"make%2:30:00::","call_ids":["senseval3.d000.s058.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":26,"predicted_sense_key":"make%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2164,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":147},"cost":{"currency":"USD","total_usd":0.015710000000000002,"input_uncached_usd":0.010820000000000001,"input_cached_usd":0.0,"output_usd":0.00489,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8177156670717523}
{"item_id":"senseval3.d000.s059.t000","gold_sense_keys":["suicide%1:04:00::"],"candidates":[{"index":1,"sense_key":"suicide%1:18:00::","synset_id":"suicide.n.02"},{"index":2,"sense_key":"suicide%1:04:00::","synset_id":"suicide.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"suicide%1:04:00::","call_ids":["senseval3.d000.s059.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"suicide%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":278,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.0032500000000000003,"input_uncached_usd":0.0013900000000000002,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8215039169881493}
{"item_id":"senseval3.d000.s059.t001","gold_sense_keys":["look%2:39:01::"],"candidates":[{"index":1,"sense_key":"look%2:39:02::","synset_id":"search.v.02"},{"index":2,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":3,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":4,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":5,"sense_key":"look%2:42:00::","synset_id":"front.v.01"},{"index":6,"sense_key":"look%2:39:01::","synset_id":"look.v.02"},{"index":7,"sense_key":"look%2:31:02::","synset_id":"count.v.08"},{"index":8,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":9,"sense_key":"look%2:29:00::","synset_id":"look.v.03"},{"index":10,"sense_key":"look%2:39:00::","synset_id":"look.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"look%2:39:01::","call_ids":["senseval3.d000.s059.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"look%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.00648,"input_uncached_usd":0.0032400000000000003,"input_cached_usd":0.0,"output_usd":0.0032400000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.330841875053011}
{"item_id":"senseval3.d000.s060.t000","gold_sense_keys":["trouble%1:04:02::","trouble%1:09:00::","trouble%1:11:00::"],"candidates":[{"index":1,"sense_key":"trouble%1:11:01::","synset_id":"fuss.n.02"},{"index":2,"sense_key":"trouble%1:11:00::","synset_id":"trouble.n.03"},{"index":3,"sense_key":"trouble%1:09:00::","synset_id":"trouble.n.01"},{"index":4,"sense_key":"trouble%1:26:02::","synset_id":"trouble.n.06"},{"index":5,"sense_key":"trouble%1:12:03::","synset_id":"worry.n.02"},{"index":6,"sense_key":"trouble%1:04:02::","synset_id":"trouble.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"trouble%1:09:00::","call_ids":["senseval3.d000.s060.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"trouble%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":493,"reasoning_output_tokens":477},"cost":{"currency":"USD","total_usd":0.017025,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.014790000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.110869291005656}
{"item_id":"senseval3.d000.s061.t001","gold_sense_keys":["fly%2:38:02::"],"candidates":[{"index":1,"sense_key":"fly%2:30:01::","synset_id":"fly.v.07"},{"index":2,"sense_key":"fly%2:35:01::","synset_id":"fly.v.13"},{"index":3,"sense_key":"fly%2:38:05::","synset_id":"fly.v.09"},{"index":4,"sense_key":"fly%2:38:07::","synset_id":"fly.v.05"},{"index":5,"sense_key":"fly%2:38:00::","synset_id":"fly.v.01"},{"index":6,"sense_key":"fly%2:38:02::","synset_id":"fly.v.02"},{"index":7,"sense_key":"fly%2:38:03::","synset_id":"flee.v.01"},{"index":8,"sense_key":"fly%2:39:00::","synset_id":"fly.v.10"},{"index":9,"sense_key":"fly%2:38:08::","synset_id":"fly.v.06"},{"index":10,"sense_key":"fly%2:38:01::","synset_id":"fly.v.03"},{"index":11,"sense_key":"fly%2:35:00::","synset_id":"fly.v.04"},{"index":12,"sense_key":"fly%2:38:06::","synset_id":"fly.v.08"},{"index":13,"sense_key":"fly%2:38:04::","synset_id":"fly.v.12"},{"index":14,"sense_key":"fly%2:30:00::","synset_id":"vanish.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"fly%2:38:02::","call_ids":["senseval3.d000.s061.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"fly%2:38:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0053750000000000004,"input_uncached_usd":0.0034850000000000003,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.968998625059612}
{"item_id":"senseval3.d000.s063.t000","gold_sense_keys":["smile%2:29:00::"],"candidates":[{"index":1,"sense_key":"smile%2:29:00::","synset_id":"smile.v.01"},{"index":2,"sense_key":"smile%2:32:00::","synset_id":"smile.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"smile%2:29:00::","call_ids":["senseval3.d000.s063.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"smile%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":264,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.00393,"input_uncached_usd":0.0013200000000000002,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.129907250055112}
{"item_id":"senseval3.d000.s064.t001","gold_sense_keys":["say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":2,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":3,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":4,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":5,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":6,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":7,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":8,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":9,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":10,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":11,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval3.d000.s064.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"say%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":879,"reasoning_output_tokens":858},"cost":{"currency":"USD","total_usd":0.029834999999999997,"input_uncached_usd":0.003465,"input_cached_usd":0.0,"output_usd":0.02637,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":18.671270666993223}
{"item_id":"senseval3.d000.s065.t000","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":2,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"},{"index":3,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":4,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":5,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":6,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"minute%1:28:00::","call_ids":["senseval3.d000.s065.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.004785000000000001,"input_uncached_usd":0.0023850000000000004,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3640362080186605}
{"item_id":"senseval3.d000.s065.t001","gold_sense_keys":["make%2:36:12::"],"candidates":[{"index":1,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":2,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":3,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":4,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":5,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":8,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":9,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":10,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":11,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":12,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":13,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":14,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":15,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":16,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":17,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":18,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":19,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":20,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":21,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":22,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":23,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":24,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":25,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":26,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":27,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":28,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":29,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":30,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":31,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":32,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":33,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":34,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":35,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":36,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":37,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":38,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":39,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":40,"sense_key":"make%2:29:01::","synset_id":"make.v.49"},{"index":41,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":42,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":43,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":44,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":45,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":46,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":47,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":48,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":49,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":37,"chosen_sense_key":"make%2:36:12::","call_ids":["senseval3.d000.s065.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":37,"predicted_sense_key":"make%2:36:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2190,"cached_input_tokens":0,"output_tokens":437,"reasoning_output_tokens":421},"cost":{"currency":"USD","total_usd":0.02406,"input_uncached_usd":0.010950000000000001,"input_cached_usd":0.0,"output_usd":0.01311,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.73920516599901}
{"item_id":"senseval3.d000.s065.t003","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s065.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":296,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":95},"cost":{"currency":"USD","total_usd":0.00481,"input_uncached_usd":0.0014800000000000002,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.830665334011428}
{"item_id":"senseval3.d000.s065.t004","gold_sense_keys":["stand_up%2:38:00::"],"candidates":[{"index":1,"sense_key":"stand_up%2:33:00::","synset_id":"stand_up.v.02"},{"index":2,"sense_key":"stand_up%2:35:00::","synset_id":"stand.v.01"},{"index":3,"sense_key":"stand_up%2:42:00::","synset_id":"stand_up.v.06"},{"index":4,"sense_key":"stand_up%2:32:00::","synset_id":"stand_up.v.05"},{"index":5,"sense_key":"stand_up%2:35:03::","synset_id":"stand.v.10"},{"index":6,"sense_key":"stand_up%2:38:00::","synset_id":"arise.v.03"},{"index":7,"sense_key":"stand_up%2:38:06::","synset_id":"bristle.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"stand_up%2:38:00::","call_ids":["senseval3.d000.s065.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"stand_up%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":540,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00462,"input_uncached_usd":0.0027,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5216312910197303}
{"item_id":"senseval3.d000.s065.t005","gold_sense_keys":["nod%2:32:01::"],"candidates":[{"index":1,"sense_key":"nod%2:29:03::","synset_id":"nod.v.03"},{"index":2,"sense_key":"nod%2:29:02::","synset_id":"nod.v.05"},{"index":3,"sense_key":"nod%2:32:01::","synset_id":"nod.v.01"},{"index":4,"sense_key":"nod%2:38:00::","synset_id":"nod.v.04"},{"index":5,"sense_key":"nod%2:32:00::","synset_id":"nod.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"nod%2:32:01::","call_ids":["senseval3.d000.s065.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"nod%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":85},"cost":{"currency":"USD","total_usd":0.005125,"input_uncached_usd":0.002095,"input_cached_usd":0.0,"output_usd":0.00303,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4961160420207307}
{"item_id":"senseval3.d000.s065.t006","gold_sense_keys":["farewell%1:10:00::"],"candidates":[{"index":1,"sense_key":"farewell%1:10:00::","synset_id":"farewell.n.01"},{"index":2,"sense_key":"farewell%1:04:00::","synset_id":"farewell.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"farewell%1:10:00::","call_ids":["senseval3.d000.s065.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"farewell%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":313,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.0036950000000000004,"input_uncached_usd":0.0015650000000000002,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8156394999241456}
{"item_id":"senseval3.d000.s066.t000","gold_sense_keys":["watch%2:39:00::","watch%2:39:02::","watch%2:41:00::"],"candidates":[{"index":1,"sense_key":"watch%2:32:09::","synset_id":"determine.v.08"},{"index":2,"sense_key":"watch%2:39:00::","synset_id":"watch.v.01"},{"index":3,"sense_key":"watch%2:39:02::","synset_id":"watch.v.04"},{"index":4,"sense_key":"watch%2:39:03::","synset_id":"watch.v.06"},{"index":5,"sense_key":"watch%2:39:01::","synset_id":"watch.v.05"},{"index":6,"sense_key":"watch%2:39:05::","synset_id":"watch.v.03"},{"index":7,"sense_key":"watch%2:41:00::","synset_id":"watch.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"watch%2:39:02::","call_ids":["senseval3.d000.s066.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"watch%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.034214999999999995,"input_uncached_usd":0.0028350000000000003,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":15.527886708034202}
{"item_id":"senseval3.d000.s066.t001","gold_sense_keys":["small%3:00:00::"],"candidates":[{"index":1,"sense_key":"small%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":2,"sense_key":"small%5:00:00:limited:00","synset_id":"minor.s.03"},{"index":3,"sense_key":"small%5:00:00:young:00","synset_id":"little.s.01"},{"index":4,"sense_key":"small%5:00:00:soft:04","synset_id":"little.s.03"},{"index":5,"sense_key":"small%5:00:00:fine:00","synset_id":"small.s.07"},{"index":6,"sense_key":"small%5:00:00:decreased:00","synset_id":"belittled.s.01"},{"index":7,"sense_key":"small%5:00:00:little:03","synset_id":"small.s.03"},{"index":8,"sense_key":"small%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":9,"sense_key":"small%5:00:00:moderate:00","synset_id":"modest.s.01"},{"index":10,"sense_key":"small%3:00:00::","synset_id":"small.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"small%3:00:00::","call_ids":["senseval3.d000.s066.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"small%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":94},"cost":{"currency":"USD","total_usd":0.006775,"input_uncached_usd":0.0034750000000000002,"input_cached_usd":0.0,"output_usd":0.0033,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.458939249976538}
{"item_id":"senseval3.d000.s066.t002","gold_sense_keys":["wiry%5:00:00:thin:03"],"candidates":[{"index":1,"sense_key":"wiry%5:00:00:hairy:00","synset_id":"wiry.s.02"},{"index":2,"sense_key":"wiry%5:00:00:thin:03","synset_id":"stringy.s.01"},{"index":3,"sense_key":"wiry%3:01:00::","synset_id":"wiry.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"wiry%5:00:00:thin:03","call_ids":["senseval3.d000.s066.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"wiry%5:00:00:thin:03","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":343,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.003215,"input_uncached_usd":0.0017150000000000002,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3012124589877203}
{"item_id":"senseval3.d000.s066.t003","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":6,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":7,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":8,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":9,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":10,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s066.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":794,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.005770000000000001,"input_uncached_usd":0.0039700000000000004,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6000009999843314}
{"item_id":"senseval3.d000.s066.t004","gold_sense_keys":["slip%2:38:01::"],"candidates":[{"index":1,"sense_key":"slip%2:38:02::","synset_id":"slip.v.08"},{"index":2,"sense_key":"slip%2:38:05::","synset_id":"slip.v.05"},{"index":3,"sense_key":"slip%2:38:01::","synset_id":"steal.v.02"},{"index":4,"sense_key":"slip%2:38:00::","synset_id":"skid.v.04"},{"index":5,"sense_key":"slip%2:31:01::","synset_id":"slip.v.10"},{"index":6,"sense_key":"slip%2:30:04::","synset_id":"slip.v.02"},{"index":7,"sense_key":"slip%2:38:12::","synset_id":"slip.v.09"},{"index":8,"sense_key":"slip%2:30:00::","synset_id":"slip.v.04"},{"index":9,"sense_key":"slip%2:40:10::","synset_id":"slip.v.07"},{"index":10,"sense_key":"slip%2:30:03::","synset_id":"dislocate.v.01"},{"index":11,"sense_key":"slip%2:31:00::","synset_id":"err.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"slip%2:38:01::","call_ids":["senseval3.d000.s066.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"slip%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.0068850000000000005,"input_uncached_usd":0.0034050000000000005,"input_cached_usd":0.0,"output_usd":0.00348,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":129.72992641699966}
{"item_id":"senseval3.d000.s066.t005","gold_sense_keys":["door%1:06:01::"],"candidates":[{"index":1,"sense_key":"door%1:06:00::","synset_id":"door.n.01"},{"index":2,"sense_key":"door%1:06:03::","synset_id":"door.n.05"},{"index":3,"sense_key":"door%1:07:00::","synset_id":"door.n.03"},{"index":4,"sense_key":"door%1:06:04::","synset_id":"door.n.04"},{"index":5,"sense_key":"door%1:06:01::","synset_id":"doorway.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"door%1:06:01::","call_ids":["senseval3.d000.s066.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"door%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":139},"cost":{"currency":"USD","total_usd":0.00713,"input_uncached_usd":0.00251,"input_cached_usd":0.0,"output_usd":0.00462,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.048939208034426}
{"item_id":"senseval3.d000.s066.t006","gold_sense_keys":["feel%2:29:00::","feel%2:31:01::","feel%2:37:00::"],"candidates":[{"index":1,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":2,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":3,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":4,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":5,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":6,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":7,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":8,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"},{"index":9,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":10,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":11,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":12,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":13,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"feel%2:37:00::","call_ids":["senseval3.d000.s066.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"feel%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":802,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01985,"input_uncached_usd":0.0040100000000000005,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.31181208405178}
{"item_id":"senseval3.d000.s066.t008","gold_sense_keys":["see%2:31:02::","see%2:32:00::","see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":2,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":3,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":6,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":7,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":8,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"},{"index":9,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":10,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":11,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":12,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":13,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":19,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":20,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":21,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":22,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":23,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":24,"sense_key":"see%2:33:00::","synset_id":"see.v.23"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"see%2:31:02::","call_ids":["senseval3.d000.s066.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"see%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1401,"cached_input_tokens":0,"output_tokens":321,"reasoning_output_tokens":305},"cost":{"currency":"USD","total_usd":0.016635,"input_uncached_usd":0.007005000000000001,"input_cached_usd":0.0,"output_usd":0.00963,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":326.176000416046}
{"item_id":"senseval3.d000.s066.t009","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"},{"index":4,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval3.d000.s066.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seem%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":1045,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.033555,"input_uncached_usd":0.0022050000000000004,"input_cached_usd":0.0,"output_usd":0.03135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":15.979093083064072}
{"item_id":"senseval3.d000.s066.t010","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":2,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d000.s066.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":458,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":136},"cost":{"currency":"USD","total_usd":0.0068200000000000005,"input_uncached_usd":0.0022900000000000004,"input_cached_usd":0.0,"output_usd":0.00453,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9311042909976095}
{"item_id":"senseval3.d000.s066.t011","gold_sense_keys":["departure%1:04:00::"],"candidates":[{"index":1,"sense_key":"departure%1:11:01::","synset_id":"deviation.n.01"},{"index":2,"sense_key":"departure%1:04:00::","synset_id":"departure.n.01"},{"index":3,"sense_key":"departure%1:11:00::","synset_id":"passing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"departure%1:04:00::","call_ids":["senseval3.d000.s066.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"departure%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":375,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.003645,"input_uncached_usd":0.0018750000000000001,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.562265625107102}
{"item_id":"senseval3.d000.s067.t000","gold_sense_keys":["decide%2:31:00::"],"candidates":[{"index":1,"sense_key":"decide%2:31:06::","synset_id":"decide.v.04"},{"index":2,"sense_key":"decide%2:31:02::","synset_id":"decide.v.03"},{"index":3,"sense_key":"decide%2:31:00::","synset_id":"decide.v.01"},{"index":4,"sense_key":"decide%2:31:01::","synset_id":"decide.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"decide%2:31:00::","call_ids":["senseval3.d000.s067.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"decide%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00394,"input_uncached_usd":0.00223,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3737506249453872}
{"item_id":"senseval3.d000.s067.t001","gold_sense_keys":["stay%2:38:00::","stay%2:42:01::","stay%2:42:02::"],"candidates":[{"index":1,"sense_key":"stay%2:42:04::","synset_id":"persist.v.03"},{"index":2,"sense_key":"stay%2:42:01::","synset_id":"bide.v.01"},{"index":3,"sense_key":"stay%2:38:01::","synset_id":"stay.v.05"},{"index":4,"sense_key":"stay%2:30:01::","synset_id":"stay.v.06"},{"index":5,"sense_key":"stay%2:42:02::","synset_id":"stay.v.04"},{"index":6,"sense_key":"stay%2:41:00::","synset_id":"stay.v.09"},{"index":7,"sense_key":"stay%2:38:00::","synset_id":"stay.v.02"},{"index":8,"sense_key":"stay%2:35:00::","synset_id":"stay.v.10"},{"index":9,"sense_key":"stay%2:42:03::","synset_id":"last_out.v.01"},{"index":10,"sense_key":"stay%2:34:00::","synset_id":"quell.v.02"},{"index":11,"sense_key":"stay%2:30:00::","synset_id":"stay.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stay%2:42:01::","call_ids":["senseval3.d000.s067.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stay%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":183,"reasoning_output_tokens":167},"cost":{"currency":"USD","total_usd":0.009115,"input_uncached_usd":0.003625,"input_cached_usd":0.0,"output_usd":0.00549,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5326623748987913}
{"item_id":"senseval3.d000.s067.t002","gold_sense_keys":["time%1:03:00::"],"candidates":[{"index":1,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":2,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":3,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":4,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":5,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":6,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":7,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":8,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":9,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"time%1:03:00::","call_ids":["senseval3.d000.s067.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"time%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":1017,"reasoning_output_tokens":995},"cost":{"currency":"USD","total_usd":0.033915,"input_uncached_usd":0.0034050000000000005,"input_cached_usd":0.0,"output_usd":0.030510000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":17.497909625060856}
{"item_id":"senseval3.d000.s067.t003","gold_sense_keys":["pass%2:38:03::"],"candidates":[{"index":1,"sense_key":"pass%2:29:00::","synset_id":"excrete.v.01"},{"index":2,"sense_key":"pass%2:40:01::","synset_id":"pass.v.20"},{"index":3,"sense_key":"pass%2:38:00::","synset_id":"pass.v.01"},{"index":4,"sense_key":"pass%2:41:02::","synset_id":"legislate.v.01"},{"index":5,"sense_key":"pass%2:38:01::","synset_id":"pass.v.07"},{"index":6,"sense_key":"pass%2:32:01::","synset_id":"communicate.v.01"},{"index":7,"sense_key":"pass%2:38:02::","synset_id":"sink.v.03"},{"index":8,"sense_key":"pass%2:35:00::","synset_id":"guide.v.05"},{"index":9,"sense_key":"pass%2:42:08::","synset_id":"run.v.03"},{"index":10,"sense_key":"pass%2:42:04::","synset_id":"exceed.v.02"},{"index":11,"sense_key":"pass%2:30:00::","synset_id":"happen.v.01"},{"index":12,"sense_key":"pass%2:41:07::","synset_id":"pass.v.16"},{"index":13,"sense_key":"pass%2:38:05::","synset_id":"travel_by.v.01"},{"index":14,"sense_key":"pass%2:41:08::","synset_id":"pass.v.14"},{"index":15,"sense_key":"pass%2:40:02::","synset_id":"pass.v.18"},{"index":16,"sense_key":"pass%2:40:00::","synset_id":"pass.v.05"},{"index":17,"sense_key":"pass%2:42:00::","synset_id":"spend.v.01"},{"index":18,"sense_key":"pass%2:41:04::","synset_id":"pass.v.09"},{"index":19,"sense_key":"pass%2:38:03::","synset_id":"elapse.v.01"},{"index":20,"sense_key":"pass%2:30:11::","synset_id":"die.v.01"},{"index":21,"sense_key":"pass%2:32:00::","synset_id":"authorize.v.01"},{"index":22,"sense_key":"pass%2:30:01::","synset_id":"evanesce.v.01"},{"index":23,"sense_key":"pass%2:40:13::","synset_id":"fall.v.21"},{"index":24,"sense_key":"pass%2:38:04::","synset_id":"pass.v.22"},{"index":25,"sense_key":"pass%2:41:11::","synset_id":"pass.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"pass%2:38:03::","call_ids":["senseval3.d000.s067.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"pass%2:38:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1415,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.009355,"input_uncached_usd":0.0070750000000000006,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6770114580867812}
{"item_id":"senseval3.d000.s067.t004","gold_sense_keys":["seem%2:39:01::"],"candidates":[{"index":1,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"seem%2:39:01::","call_ids":["senseval3.d000.s067.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"seem%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":405,"reasoning_output_tokens":389},"cost":{"currency":"USD","total_usd":0.01445,"input_uncached_usd":0.0023000000000000004,"input_cached_usd":0.0,"output_usd":0.012150000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.341624458087608}
{"item_id":"senseval3.d000.s067.t005","gold_sense_keys":["strange%3:00:00::"],"candidates":[{"index":1,"sense_key":"strange%5:00:01:unfamiliar:00","synset_id":"strange.s.01"},{"index":2,"sense_key":"strange%3:00:00::","synset_id":"strange.a.01"},{"index":3,"sense_key":"strange%3:00:01::","synset_id":"foreign.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"strange%3:00:00::","call_ids":["senseval3.d000.s067.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"strange%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":499,"reasoning_output_tokens":483},"cost":{"currency":"USD","total_usd":0.01708,"input_uncached_usd":0.0021100000000000003,"input_cached_usd":0.0,"output_usd":0.01497,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.113914291956462}
{"item_id":"senseval3.d000.s067.t006","gold_sense_keys":["little%3:00:00::","little%3:00:01::"],"candidates":[{"index":1,"sense_key":"little%3:00:03::","synset_id":"little.a.02"},{"index":2,"sense_key":"little%5:00:00:unimportant:00","synset_id":"fiddling.s.01"},{"index":3,"sense_key":"little%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":4,"sense_key":"little%3:00:00::","synset_id":"short.a.03"},{"index":5,"sense_key":"little%3:00:01::","synset_id":"small.a.01"},{"index":6,"sense_key":"little%5:00:00:soft:04","synset_id":"little.s.03"},{"index":7,"sense_key":"little%5:00:00:young:00","synset_id":"little.s.01"},{"index":8,"sense_key":"little%5:00:00:emotional:00","synset_id":"little.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"little%3:00:00::","call_ids":["senseval3.d000.s067.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"little%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":222,"reasoning_output_tokens":206},"cost":{"currency":"USD","total_usd":0.009965,"input_uncached_usd":0.003305,"input_cached_usd":0.0,"output_usd":0.00666,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.007698334055021}
{"item_id":"senseval3.d000.s067.t007","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":2,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":3,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":4,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:05:00::","synset_id":"world.n.08"},{"index":7,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":10,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":11,"sense_key":"man%1:15:00::","synset_id":"man.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s067.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":813,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.005715,"input_uncached_usd":0.004065,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7017252499936149}
{"item_id":"senseval3.d000.s067.t008","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":2,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":3,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":4,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d000.s067.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.00498,"input_uncached_usd":0.0022500000000000003,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2715978749329224}
{"item_id":"senseval3.d000.s067.t009","gold_sense_keys":["glass%1:06:00::"],"candidates":[{"index":1,"sense_key":"glass%1:27:00::","synset_id":"glass.n.01"},{"index":2,"sense_key":"glass%1:06:01::","synset_id":"methamphetamine.n.01"},{"index":3,"sense_key":"glass%1:06:00::","synset_id":"glass.n.02"},{"index":4,"sense_key":"glass%1:23:00::","synset_id":"glass.n.03"},{"index":5,"sense_key":"glass%1:06:02::","synset_id":"looking_glass.n.01"},{"index":6,"sense_key":"glass%1:06:03::","synset_id":"field_glass.n.01"},{"index":7,"sense_key":"glass%1:06:04::","synset_id":"glass.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"glass%1:06:00::","call_ids":["senseval3.d000.s067.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"glass%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.0049900000000000005,"input_uncached_usd":0.0025900000000000003,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.33339054195676}
{"item_id":"senseval3.d000.s067.t010","gold_sense_keys":["table%1:06:01::"],"candidates":[{"index":1,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":2,"sense_key":"table%1:14:01::","synset_id":"table.n.05"},{"index":3,"sense_key":"table%1:06:02::","synset_id":"table.n.03"},{"index":4,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":5,"sense_key":"table%1:13:01::","synset_id":"board.n.04"},{"index":6,"sense_key":"table%1:14:00::","synset_id":"table.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"table%1:06:01::","call_ids":["senseval3.d000.s067.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"table%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":237,"reasoning_output_tokens":221},"cost":{"currency":"USD","total_usd":0.00973,"input_uncached_usd":0.0026200000000000004,"input_cached_usd":0.0,"output_usd":0.00711,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.035861624986865}
{"item_id":"senseval3.d000.s068.t002","gold_sense_keys":["return%2:38:00::"],"candidates":[{"index":1,"sense_key":"return%2:32:02::","synset_id":"return.v.16"},{"index":2,"sense_key":"return%2:40:08::","synset_id":"render.v.07"},{"index":3,"sense_key":"return%2:32:01::","synset_id":"retort.v.01"},{"index":4,"sense_key":"return%2:36:00::","synset_id":"render.v.04"},{"index":5,"sense_key":"return%2:38:00::","synset_id":"return.v.01"},{"index":6,"sense_key":"return%2:40:03::","synset_id":"return.v.06"},{"index":7,"sense_key":"return%2:30:01::","synset_id":"come_back.v.01"},{"index":8,"sense_key":"return%2:38:01::","synset_id":"return.v.14"},{"index":9,"sense_key":"return%2:35:01::","synset_id":"return.v.07"},{"index":10,"sense_key":"return%2:40:13::","synset_id":"fall.v.21"},{"index":11,"sense_key":"return%2:30:00::","synset_id":"revert.v.01"},{"index":12,"sense_key":"return%2:32:00::","synset_id":"hark_back.v.01"},{"index":13,"sense_key":"return%2:38:12::","synset_id":"return.v.05"},{"index":14,"sense_key":"return%2:32:03::","synset_id":"render.v.05"},{"index":15,"sense_key":"return%2:40:00::","synset_id":"refund.v.01"},{"index":16,"sense_key":"return%2:41:00::","synset_id":"reelect.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"return%2:38:00::","call_ids":["senseval3.d000.s068.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"return%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":889,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.006275,"input_uncached_usd":0.004445,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.459540749900043}
{"item_id":"senseval3.d000.s068.t005","gold_sense_keys":["put%2:35:00::"],"candidates":[{"index":1,"sense_key":"put%2:32:00::","synset_id":"frame.v.04"},{"index":2,"sense_key":"put%2:39:13::","synset_id":"put.v.07"},{"index":3,"sense_key":"put%2:40:00::","synset_id":"invest.v.01"},{"index":4,"sense_key":"put%2:31:13::","synset_id":"place.v.11"},{"index":5,"sense_key":"put%2:36:13::","synset_id":"put.v.08"},{"index":6,"sense_key":"put%2:35:00::","synset_id":"put.v.01"},{"index":7,"sense_key":"put%2:31:00::","synset_id":"arrange.v.07"},{"index":8,"sense_key":"put%2:34:00::","synset_id":"put.v.04"},{"index":9,"sense_key":"put%2:35:05::","synset_id":"put.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"put%2:35:00::","call_ids":["senseval3.d000.s068.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"put%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01918,"input_uncached_usd":0.00334,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.070200541056693}
{"item_id":"senseval3.d000.s068.t006","gold_sense_keys":["whole%3:00:00::"],"candidates":[{"index":1,"sense_key":"whole%3:00:02::","synset_id":"whole.a.02"},{"index":2,"sense_key":"whole%5:00:00:undiversified:00","synset_id":"solid.s.13"},{"index":3,"sense_key":"whole%5:00:00:uninjured:00","synset_id":"unharmed.s.01"},{"index":4,"sense_key":"whole%3:00:00::","synset_id":"whole.a.01"},{"index":5,"sense_key":"whole%5:00:00:healthy:00","synset_id":"hale.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"whole%3:00:00::","call_ids":["senseval3.d000.s068.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"whole%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.00407,"input_uncached_usd":0.00251,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9566420409828424}
{"item_id":"senseval3.d000.s068.t007","gold_sense_keys":["incident%1:11:00::"],"candidates":[{"index":1,"sense_key":"incident%1:11:00::","synset_id":"incident.n.01"},{"index":2,"sense_key":"incident%1:26:00::","synset_id":"incident.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"incident%1:11:00::","call_ids":["senseval3.d000.s068.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"incident%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":344,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0036100000000000004,"input_uncached_usd":0.0017200000000000002,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8876409580698237}
{"item_id":"senseval3.d000.s068.t008","gold_sense_keys":["mind%1:09:00::","mind%1:09:01::","mind%1:09:03::"],"candidates":[{"index":1,"sense_key":"mind%1:09:03::","synset_id":"mind.n.05"},{"index":2,"sense_key":"mind%1:09:06::","synset_id":"mind.n.06"},{"index":3,"sense_key":"mind%1:09:05::","synset_id":"mind.n.07"},{"index":4,"sense_key":"mind%1:09:00::","synset_id":"mind.n.01"},{"index":5,"sense_key":"mind%1:18:00::","synset_id":"thinker.n.01"},{"index":6,"sense_key":"mind%1:09:04::","synset_id":"judgment.n.01"},{"index":7,"sense_key":"mind%1:09:01::","synset_id":"mind.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"mind%1:09:00::","call_ids":["senseval3.d000.s068.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"mind%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":375,"reasoning_output_tokens":360},"cost":{"currency":"USD","total_usd":0.014175000000000002,"input_uncached_usd":0.002925,"input_cached_usd":0.0,"output_usd":0.01125,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.327802333049476}
{"item_id":"senseval3.d000.s069.t000","gold_sense_keys":["next%5:00:00:succeeding:00"],"candidates":[{"index":1,"sense_key":"next%5:00:00:close:01","synset_id":"adjacent.s.01"},{"index":2,"sense_key":"next%5:00:00:incoming:00","synset_id":"future.s.02"},{"index":3,"sense_key":"next%5:00:00:succeeding:00","synset_id":"following.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"next%5:00:00:succeeding:00","call_ids":["senseval3.d000.s069.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"next%5:00:00:succeeding:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":43,"reasoning_output_tokens":28},"cost":{"currency":"USD","total_usd":0.0035,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2537625000113621}
{"item_id":"senseval3.d000.s069.t001","gold_sense_keys":["day%1:28:00::"],"candidates":[{"index":1,"sense_key":"day%1:28:03::","synset_id":"day.n.02"},{"index":2,"sense_key":"day%1:28:00::","synset_id":"day.n.01"},{"index":3,"sense_key":"day%1:28:02::","synset_id":"day.n.04"},{"index":4,"sense_key":"day%1:28:07::","synset_id":"day.n.07"},{"index":5,"sense_key":"day%1:28:04::","synset_id":"day.n.05"},{"index":6,"sense_key":"day%1:26:00::","synset_id":"day.n.09"},{"index":7,"sense_key":"day%1:28:06::","synset_id":"sidereal_day.n.01"},{"index":8,"sense_key":"day%1:28:05::","synset_id":"day.n.06"},{"index":9,"sense_key":"day%1:28:01::","synset_id":"day.n.03"},{"index":10,"sense_key":"day%1:18:00::","synset_id":"day.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"day%1:28:00::","call_ids":["senseval3.d000.s069.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"day%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01969,"input_uncached_usd":0.00385,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.458046125015244}
{"item_id":"senseval3.d000.s069.t002","gold_sense_keys":["hangover%1:26:00::"],"candidates":[{"index":1,"sense_key":"hangover%1:04:00::","synset_id":"hangover.n.03"},{"index":2,"sense_key":"hangover%1:26:00::","synset_id":"hangover.n.01"},{"index":3,"sense_key":"hangover%1:18:00::","synset_id":"holdover.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"hangover%1:26:00::","call_ids":["senseval3.d000.s069.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"hangover%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00385,"input_uncached_usd":0.00214,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6170174590079114}
{"item_id":"senseval3.d000.s069.t003","gold_sense_keys":["remind%2:31:00::"],"candidates":[{"index":1,"sense_key":"remind%2:31:00::","synset_id":"remind.v.01"},{"index":2,"sense_key":"remind%2:32:00::","synset_id":"prompt.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"remind%2:31:00::","call_ids":["senseval3.d000.s069.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"remind%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":386,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.004030000000000001,"input_uncached_usd":0.00193,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5928939590230584}
{"item_id":"senseval3.d000.s069.t004","gold_sense_keys":["night%1:28:03::"],"candidates":[{"index":1,"sense_key":"night%1:26:00::","synset_id":"night.n.05"},{"index":2,"sense_key":"night%1:28:05::","synset_id":"night.n.06"},{"index":3,"sense_key":"night%1:28:00::","synset_id":"night.n.01"},{"index":4,"sense_key":"night%1:28:03::","synset_id":"night.n.07"},{"index":5,"sense_key":"night%1:28:04::","synset_id":"night.n.03"},{"index":6,"sense_key":"night%1:28:01::","synset_id":"night.n.02"},{"index":7,"sense_key":"night%1:18:00::","synset_id":"nox.n.01"},{"index":8,"sense_key":"night%1:28:02::","synset_id":"night.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"night%1:28:02::","call_ids":["senseval3.d000.s069.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"night%1:28:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":558,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01863,"input_uncached_usd":0.0027900000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.244543957989663}
{"item_id":"senseval3.d000.s070.t000","gold_sense_keys":["hangover%1:26:00::"],"candidates":[{"index":1,"sense_key":"hangover%1:26:00::","synset_id":"hangover.n.01"},{"index":2,"sense_key":"hangover%1:18:00::","synset_id":"holdover.n.01"},{"index":3,"sense_key":"hangover%1:04:00::","synset_id":"hangover.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hangover%1:26:00::","call_ids":["senseval3.d000.s070.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hangover%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":33,"reasoning_output_tokens":17},"cost":{"currency":"USD","total_usd":0.003295,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.846870500012301}
{"item_id":"senseval3.d000.s070.t001","gold_sense_keys":["remind%2:31:00::"],"candidates":[{"index":1,"sense_key":"remind%2:32:00::","synset_id":"prompt.v.03"},{"index":2,"sense_key":"remind%2:31:00::","synset_id":"remind.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"remind%2:31:00::","call_ids":["senseval3.d000.s070.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"remind%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.004135,"input_uncached_usd":0.002095,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7522012499393895}
{"item_id":"senseval3.d000.s071.t000","gold_sense_keys":["go%2:38:00::"],"candidates":[{"index":1,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":2,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":3,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"},{"index":4,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":5,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":6,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":7,"sense_key":"go%2:42:08::","synset_id":"go.v.25"},{"index":8,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":9,"sense_key":"go%2:30:08::","synset_id":"go.v.09"},{"index":10,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":11,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":12,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":13,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":14,"sense_key":"go%2:38:01::","synset_id":"go.v.03"},{"index":15,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"},{"index":16,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":17,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":18,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":19,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":20,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":21,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":22,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":23,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":24,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":25,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":26,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":27,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"},{"index":28,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":29,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":30,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"go%2:38:00::","call_ids":["senseval3.d000.s071.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"go%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1724,"cached_input_tokens":0,"output_tokens":191,"reasoning_output_tokens":175},"cost":{"currency":"USD","total_usd":0.014350000000000002,"input_uncached_usd":0.008620000000000001,"input_cached_usd":0.0,"output_usd":0.00573,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.02793074992951}
{"item_id":"senseval3.d000.s071.t002","gold_sense_keys":["day%1:28:04::"],"candidates":[{"index":1,"sense_key":"day%1:28:05::","synset_id":"day.n.06"},{"index":2,"sense_key":"day%1:28:03::","synset_id":"day.n.02"},{"index":3,"sense_key":"day%1:28:07::","synset_id":"day.n.07"},{"index":4,"sense_key":"day%1:28:04::","synset_id":"day.n.05"},{"index":5,"sense_key":"day%1:28:06::","synset_id":"sidereal_day.n.01"},{"index":6,"sense_key":"day%1:28:02::","synset_id":"day.n.04"},{"index":7,"sense_key":"day%1:18:00::","synset_id":"day.n.10"},{"index":8,"sense_key":"day%1:26:00::","synset_id":"day.n.09"},{"index":9,"sense_key":"day%1:28:00::","synset_id":"day.n.01"},{"index":10,"sense_key":"day%1:28:01::","synset_id":"day.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"day%1:28:00::","call_ids":["senseval3.d000.s071.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"day%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019785,"input_uncached_usd":0.003945000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.988065332989208}
{"item_id":"senseval3.d000.s071.t003","gold_sense_keys":["pass%2:38:05::"],"candidates":[{"index":1,"sense_key":"pass%2:38:03::","synset_id":"elapse.v.01"},{"index":2,"sense_key":"pass%2:30:11::","synset_id":"die.v.01"},{"index":3,"sense_key":"pass%2:35:00::","synset_id":"guide.v.05"},{"index":4,"sense_key":"pass%2:40:02::","synset_id":"pass.v.18"},{"index":5,"sense_key":"pass%2:42:00::","synset_id":"spend.v.01"},{"index":6,"sense_key":"pass%2:41:08::","synset_id":"pass.v.14"},{"index":7,"sense_key":"pass%2:41:02::","synset_id":"legislate.v.01"},{"index":8,"sense_key":"pass%2:41:07::","synset_id":"pass.v.16"},{"index":9,"sense_key":"pass%2:41:04::","synset_id":"pass.v.09"},{"index":10,"sense_key":"pass%2:38:01::","synset_id":"pass.v.07"},{"index":11,"sense_key":"pass%2:40:00::","synset_id":"pass.v.05"},{"index":12,"sense_key":"pass%2:40:13::","synset_id":"fall.v.21"},{"index":13,"sense_key":"pass%2:42:04::","synset_id":"exceed.v.02"},{"index":14,"sense_key":"pass%2:40:01::","synset_id":"pass.v.20"},{"index":15,"sense_key":"pass%2:38:02::","synset_id":"sink.v.03"},{"index":16,"sense_key":"pass%2:41:11::","synset_id":"pass.v.17"},{"index":17,"sense_key":"pass%2:38:04::","synset_id":"pass.v.22"},{"index":18,"sense_key":"pass%2:30:01::","synset_id":"evanesce.v.01"},{"index":19,"sense_key":"pass%2:29:00::","synset_id":"excrete.v.01"},{"index":20,"sense_key":"pass%2:30:00::","synset_id":"happen.v.01"},{"index":21,"sense_key":"pass%2:38:05::","synset_id":"travel_by.v.01"},{"index":22,"sense_key":"pass%2:32:01::","synset_id":"communicate.v.01"},{"index":23,"sense_key":"pass%2:42:08::","synset_id":"run.v.03"},{"index":24,"sense_key":"pass%2:32:00::","synset_id":"authorize.v.01"},{"index":25,"sense_key":"pass%2:38:00::","synset_id":"pass.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":21,"chosen_sense_key":"pass%2:38:05::","call_ids":["senseval3.d000.s071.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":21,"predicted_sense_key":"pass%2:38:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1465,"cached_input_tokens":0,"output_tokens":161,"reasoning_output_tokens":145},"cost":{"currency":"USD","total_usd":0.012155000000000001,"input_uncached_usd":0.007325000000000001,"input_cached_usd":0.0,"output_usd":0.00483,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.62512666604016}
{"item_id":"senseval3.d000.s071.t007","gold_sense_keys":["alive%3:00:01::"],"candidates":[{"index":1,"sense_key":"alive%3:00:01::","synset_id":"alive.a.01"},{"index":2,"sense_key":"alive%3:00:04::","synset_id":"animated.a.01"},{"index":3,"sense_key":"alive%5:00:00:active:05","synset_id":"alive.s.05"},{"index":4,"sense_key":"alive%5:00:00:existent:00","synset_id":"active.s.03"},{"index":5,"sense_key":"alive%5:00:00:aware:00","synset_id":"alert.s.02"},{"index":6,"sense_key":"alive%5:00:00:sensitive:02","synset_id":"alive.s.02"},{"index":7,"sense_key":"alive%5:00:00:lively:00","synset_id":"alive.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"alive%3:00:01::","call_ids":["senseval3.d000.s071.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"alive%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.00542,"input_uncached_usd":0.00305,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9733110829256475}
{"item_id":"senseval3.d000.s071.t009","gold_sense_keys":["tenant%1:18:00::"],"candidates":[{"index":1,"sense_key":"tenant%1:18:00::","synset_id":"tenant.n.01"},{"index":2,"sense_key":"tenant%1:18:02::","synset_id":"tenant.n.02"},{"index":3,"sense_key":"tenant%1:18:01::","synset_id":"tenant.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tenant%1:18:00::","call_ids":["senseval3.d000.s071.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tenant%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004205,"input_uncached_usd":0.0021950000000000003,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.536174749955535}
{"item_id":"senseval3.d000.s071.t010","gold_sense_keys":["burden%1:09:01::"],"candidates":[{"index":1,"sense_key":"burden%1:10:00::","synset_id":"effect.n.04"},{"index":2,"sense_key":"burden%1:09:00::","synset_id":"burden.n.04"},{"index":3,"sense_key":"burden%1:09:01::","synset_id":"burden.n.01"},{"index":4,"sense_key":"burden%1:06:01::","synset_id":"load.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"burden%1:09:01::","call_ids":["senseval3.d000.s071.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"burden%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0041,"input_uncached_usd":0.00239,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8604017089819536}
{"item_id":"senseval3.d000.s071.t011","gold_sense_keys":["life%1:26:02::"],"candidates":[{"index":1,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":2,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":3,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":4,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":5,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":6,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":7,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":12,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":13,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":14,"sense_key":"life%1:28:01::","synset_id":"life.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"life%1:26:02::","call_ids":["senseval3.d000.s071.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"life%1:26:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":893,"cached_input_tokens":0,"output_tokens":268,"reasoning_output_tokens":252},"cost":{"currency":"USD","total_usd":0.012505,"input_uncached_usd":0.004465,"input_cached_usd":0.0,"output_usd":0.00804,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.750932042021304}
{"item_id":"senseval3.d000.s072.t001","gold_sense_keys":["ignore%2:32:02::"],"candidates":[{"index":1,"sense_key":"ignore%2:39:00::","synset_id":"ignore.v.03"},{"index":2,"sense_key":"ignore%2:31:01::","synset_id":"ignore.v.05"},{"index":3,"sense_key":"ignore%2:32:00::","synset_id":"dismiss.v.01"},{"index":4,"sense_key":"ignore%2:31:03::","synset_id":"neglect.v.04"},{"index":5,"sense_key":"ignore%2:32:02::","synset_id":"ignore.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"ignore%2:32:02::","call_ids":["senseval3.d000.s072.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"ignore%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":180,"reasoning_output_tokens":165},"cost":{"currency":"USD","total_usd":0.007765000000000001,"input_uncached_usd":0.0023650000000000003,"input_cached_usd":0.0,"output_usd":0.0054,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.97582012496423}
{"item_id":"senseval3.d000.s074.t001","gold_sense_keys":["alive%3:00:01::"],"candidates":[{"index":1,"sense_key":"alive%5:00:00:active:05","synset_id":"alive.s.05"},{"index":2,"sense_key":"alive%5:00:00:lively:00","synset_id":"alive.s.01"},{"index":3,"sense_key":"alive%5:00:00:sensitive:02","synset_id":"alive.s.02"},{"index":4,"sense_key":"alive%3:00:04::","synset_id":"animated.a.01"},{"index":5,"sense_key":"alive%5:00:00:existent:00","synset_id":"active.s.03"},{"index":6,"sense_key":"alive%3:00:01::","synset_id":"alive.a.01"},{"index":7,"sense_key":"alive%5:00:00:aware:00","synset_id":"alert.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"alive%3:00:01::","call_ids":["senseval3.d000.s074.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"alive%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":108},"cost":{"currency":"USD","total_usd":0.006535000000000001,"input_uncached_usd":0.0028150000000000002,"input_cached_usd":0.0,"output_usd":0.00372,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8694458750542253}
{"item_id":"senseval3.d000.s075.t001","gold_sense_keys":["mail%2:35:00::"],"candidates":[{"index":1,"sense_key":"mail%2:35:00::","synset_id":"mail.v.01"},{"index":2,"sense_key":"mail%2:32:00::","synset_id":"mail.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mail%2:35:00::","call_ids":["senseval3.d000.s075.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mail%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":365,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.004165,"input_uncached_usd":0.0018250000000000002,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7853343749884516}
{"item_id":"senseval3.d000.s075.t003","gold_sense_keys":["check%1:21:00::"],"candidates":[{"index":1,"sense_key":"check%1:09:01::","synset_id":"confirmation.n.01"},{"index":2,"sense_key":"check%1:10:00::","synset_id":"check.n.03"},{"index":3,"sense_key":"check%1:09:00::","synset_id":"assay.n.01"},{"index":4,"sense_key":"check%1:09:02::","synset_id":"hindrance.n.01"},{"index":5,"sense_key":"check%1:06:00::","synset_id":"check.n.10"},{"index":6,"sense_key":"check%1:04:01::","synset_id":"bridle.n.02"},{"index":7,"sense_key":"check%1:10:01::","synset_id":"check_mark.n.01"},{"index":8,"sense_key":"check%1:26:00::","synset_id":"arrest.n.02"},{"index":9,"sense_key":"check%1:04:05::","synset_id":"check.n.06"},{"index":10,"sense_key":"check%1:04:04::","synset_id":"check.n.13"},{"index":11,"sense_key":"check%1:07:00::","synset_id":"check.n.09"},{"index":12,"sense_key":"check%1:21:00::","synset_id":"check.n.01"},{"index":13,"sense_key":"check%1:04:02::","synset_id":"check.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"check%1:21:00::","call_ids":["senseval3.d000.s075.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"check%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":844,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.006470000000000001,"input_uncached_usd":0.004220000000000001,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8870882079936564}
{"item_id":"senseval3.d000.s075.t005","gold_sense_keys":["maintenance%1:21:01::"],"candidates":[{"index":1,"sense_key":"maintenance%1:04:01::","synset_id":"sustenance.n.03"},{"index":2,"sense_key":"maintenance%1:21:01::","synset_id":"alimony.n.01"},{"index":3,"sense_key":"maintenance%1:21:00::","synset_id":"maintenance.n.02"},{"index":4,"sense_key":"maintenance%1:04:00::","synset_id":"care.n.06"},{"index":5,"sense_key":"maintenance%1:04:02::","synset_id":"maintenance.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"maintenance%1:21:01::","call_ids":["senseval3.d000.s075.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"maintenance%1:21:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":104},"cost":{"currency":"USD","total_usd":0.0062450000000000006,"input_uncached_usd":0.002645,"input_cached_usd":0.0,"output_usd":0.0036,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5476649589836597}
{"item_id":"senseval3.d000.s075.t007","gold_sense_keys":["wish%2:32:02::","wish%2:37:00::"],"candidates":[{"index":1,"sense_key":"wish%2:32:00::","synset_id":"wish.v.06"},{"index":2,"sense_key":"wish%2:32:01::","synset_id":"wish.v.05"},{"index":3,"sense_key":"wish%2:37:01::","synset_id":"wish.v.04"},{"index":4,"sense_key":"wish%2:37:00::","synset_id":"wish.v.01"},{"index":5,"sense_key":"wish%2:37:02::","synset_id":"wish.v.02"},{"index":6,"sense_key":"wish%2:32:02::","synset_id":"wish.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"wish%2:37:00::","call_ids":["senseval3.d000.s075.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"wish%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":393,"reasoning_output_tokens":378},"cost":{"currency":"USD","total_usd":0.014225000000000002,"input_uncached_usd":0.002435,"input_cached_usd":0.0,"output_usd":0.01179,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.835414374945685}
{"item_id":"senseval3.d000.s075.t008","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s075.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":343,"cached_input_tokens":0,"output_tokens":325,"reasoning_output_tokens":309},"cost":{"currency":"USD","total_usd":0.011465,"input_uncached_usd":0.0017150000000000002,"input_cached_usd":0.0,"output_usd":0.00975,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.639571042032912}
{"item_id":"senseval3.d000.s075.t009","gold_sense_keys":["do%2:36:01::","do%2:41:01::","do%2:41:04::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":3,"sense_key":"do%2:38:00::","synset_id":"do.v.13"},{"index":4,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":5,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":6,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":7,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":8,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":9,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":10,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":11,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":12,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":13,"sense_key":"do%2:29:09::","synset_id":"act.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"do%2:36:01::","call_ids":["senseval3.d000.s075.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":877,"cached_input_tokens":0,"output_tokens":240,"reasoning_output_tokens":224},"cost":{"currency":"USD","total_usd":0.011585000000000002,"input_uncached_usd":0.004385,"input_cached_usd":0.0,"output_usd":0.0072,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.994139416958205}
{"item_id":"senseval3.d000.s076.t000","gold_sense_keys":["come%2:38:04::"],"candidates":[{"index":1,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":2,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":3,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":4,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":5,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":6,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":7,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":8,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":9,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":10,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":11,"sense_key":"come%2:31:13::","synset_id":"come.v.21"},{"index":12,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":13,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":14,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":15,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":16,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":17,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":18,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":19,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":20,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":21,"sense_key":"come%2:39:00::","synset_id":"come.v.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":20,"chosen_sense_key":"come%2:38:04::","call_ids":["senseval3.d000.s076.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":20,"predicted_sense_key":"come%2:38:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1146,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02157,"input_uncached_usd":0.005730000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.048343541915528}
{"item_id":"senseval3.d000.s076.t001","gold_sense_keys":["home%1:15:04::"],"candidates":[{"index":1,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":2,"sense_key":"home%1:15:02::","synset_id":"home.n.03"},{"index":3,"sense_key":"home%1:15:03::","synset_id":"home.n.06"},{"index":4,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"},{"index":5,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":6,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":7,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"},{"index":8,"sense_key":"home%1:06:01::","synset_id":"home.n.09"},{"index":9,"sense_key":"home%1:14:00::","synset_id":"family.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"home%1:15:04::","call_ids":["senseval3.d000.s076.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"home%1:15:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":686,"cached_input_tokens":0,"output_tokens":164,"reasoning_output_tokens":148},"cost":{"currency":"USD","total_usd":0.008350000000000002,"input_uncached_usd":0.0034300000000000003,"input_cached_usd":0.0,"output_usd":0.00492,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.175763500039466}
{"item_id":"senseval3.d000.s076.t002","gold_sense_keys":["work%1:06:01::"],"candidates":[{"index":1,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":2,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":3,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":4,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":5,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"},{"index":6,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":7,"sense_key":"work%1:09:00::","synset_id":"study.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"work%1:06:01::","call_ids":["senseval3.d000.s076.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"work%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":210,"reasoning_output_tokens":194},"cost":{"currency":"USD","total_usd":0.009465000000000001,"input_uncached_usd":0.0031650000000000003,"input_cached_usd":0.0,"output_usd":0.0063,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.61766620806884}
{"item_id":"senseval3.d000.s076.t004","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":2,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":3,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":4,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":7,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":8,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":9,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":10,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":11,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":12,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":13,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":14,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":15,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":16,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":17,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":18,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":19,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"},{"index":22,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":23,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":24,"sense_key":"see%2:31:08::","synset_id":"see.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":23,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval3.d000.s076.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":23,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1417,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.009155,"input_uncached_usd":0.007085,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9449918330647051}
{"item_id":"senseval3.d000.s077.t000","gold_sense_keys":["lobby%1:06:00::"],"candidates":[{"index":1,"sense_key":"lobby%1:06:00::","synset_id":"anteroom.n.01"},{"index":2,"sense_key":"lobby%1:14:00::","synset_id":"lobby.n.03"},{"index":3,"sense_key":"lobby%1:14:01::","synset_id":"lobby.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lobby%1:06:00::","call_ids":["senseval3.d000.s077.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lobby%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":364,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.0032600000000000003,"input_uncached_usd":0.0018200000000000002,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.670427290955558}
{"item_id":"senseval3.d000.s077.t001","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":4,"sense_key":"people%1:14:02::","synset_id":"people.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval3.d000.s077.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.00355,"input_uncached_usd":0.00217,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.748927208012901}
{"item_id":"senseval3.d000.s077.t002","gold_sense_keys":["stand%2:35:00::"],"candidates":[{"index":1,"sense_key":"stand%2:42:01::","synset_id":"stand.v.09"},{"index":2,"sense_key":"stand%2:42:06::","synset_id":"stand.v.07"},{"index":3,"sense_key":"stand%2:33:00::","synset_id":"resist.v.04"},{"index":4,"sense_key":"stand%2:35:00::","synset_id":"stand.v.01"},{"index":5,"sense_key":"stand%2:31:01::","synset_id":"stand.v.06"},{"index":6,"sense_key":"stand%2:42:02::","synset_id":"stand.v.08"},{"index":7,"sense_key":"stand%2:42:03::","synset_id":"stand.v.03"},{"index":8,"sense_key":"stand%2:35:01::","synset_id":"stand.v.10"},{"index":9,"sense_key":"stand%2:42:04::","synset_id":"stand.v.02"},{"index":10,"sense_key":"stand%2:42:00::","synset_id":"stand.v.04"},{"index":11,"sense_key":"stand%2:35:10::","synset_id":"stand.v.12"},{"index":12,"sense_key":"stand%2:31:00::","synset_id":"digest.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"stand%2:35:00::","call_ids":["senseval3.d000.s077.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"stand%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":714,"cached_input_tokens":0,"output_tokens":1284,"reasoning_output_tokens":1256},"cost":{"currency":"USD","total_usd":0.04209,"input_uncached_usd":0.0035700000000000003,"input_cached_usd":0.0,"output_usd":0.03852,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":24.601747791981325}
{"item_id":"senseval3.d000.s077.t003","gold_sense_keys":["talk%2:32:01::"],"candidates":[{"index":1,"sense_key":"talk%2:32:05::","synset_id":"lecture.v.01"},{"index":2,"sense_key":"talk%2:32:00::","synset_id":"talk.v.02"},{"index":3,"sense_key":"talk%2:32:02::","synset_id":"spill.v.05"},{"index":4,"sense_key":"talk%2:32:01::","synset_id":"talk.v.01"},{"index":5,"sense_key":"talk%2:32:06::","synset_id":"spill_the_beans.v.01"},{"index":6,"sense_key":"talk%2:32:03::","synset_id":"speak.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"talk%2:32:01::","call_ids":["senseval3.d000.s077.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"talk%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.0056300000000000005,"input_uncached_usd":0.0026300000000000004,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.45539562497288}
{"item_id":"senseval3.d000.s078.t000","gold_sense_keys":["spine%1:08:00::"],"candidates":[{"index":1,"sense_key":"spine%1:05:00::","synset_id":"spine.n.05"},{"index":2,"sense_key":"spine%1:25:00::","synset_id":"spur.n.02"},{"index":3,"sense_key":"spine%1:06:00::","synset_id":"spine.n.04"},{"index":4,"sense_key":"spine%1:20:00::","synset_id":"spine.n.03"},{"index":5,"sense_key":"spine%1:08:00::","synset_id":"spinal_column.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"spine%1:08:00::","call_ids":["senseval3.d000.s078.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"spine%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.00465,"input_uncached_usd":0.00252,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.565297082997859}
{"item_id":"senseval3.d000.s078.t001","gold_sense_keys":["crawl%2:42:01::"],"candidates":[{"index":1,"sense_key":"crawl%2:38:00::","synset_id":"crawl.v.01"},{"index":2,"sense_key":"crawl%2:38:01::","synset_id":"crawl.v.05"},{"index":3,"sense_key":"crawl%2:42:00::","synset_id":"crawl.v.03"},{"index":4,"sense_key":"crawl%2:42:01::","synset_id":"crawl.v.02"},{"index":5,"sense_key":"crawl%2:38:03::","synset_id":"fawn.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"crawl%2:42:01::","call_ids":["senseval3.d000.s078.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"crawl%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.00434,"input_uncached_usd":0.0022400000000000002,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7302089160075411}
{"item_id":"senseval3.d000.s078.t003","gold_sense_keys":["premonition%1:12:00::"],"candidates":[{"index":1,"sense_key":"premonition%1:12:00::","synset_id":"foreboding.n.01"},{"index":2,"sense_key":"premonition%1:10:00::","synset_id":"forewarning.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"premonition%1:12:00::","call_ids":["senseval3.d000.s078.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"premonition%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":350,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.0037,"input_uncached_usd":0.00175,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4774744589813054}
{"item_id":"senseval3.d000.s078.t004","gold_sense_keys":["ask%2:32:04::"],"candidates":[{"index":1,"sense_key":"ask%2:32:05::","synset_id":"ask.v.04"},{"index":2,"sense_key":"ask%2:32:00::","synset_id":"ask.v.01"},{"index":3,"sense_key":"ask%2:32:04::","synset_id":"ask.v.05"},{"index":4,"sense_key":"ask%2:32:02::","synset_id":"ask.v.03"},{"index":5,"sense_key":"ask%2:32:01::","synset_id":"ask.v.02"},{"index":6,"sense_key":"ask%2:32:09::","synset_id":"ask.v.07"},{"index":7,"sense_key":"ask%2:42:00::","synset_id":"necessitate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ask%2:32:04::","call_ids":["senseval3.d000.s078.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ask%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":312,"reasoning_output_tokens":296},"cost":{"currency":"USD","total_usd":0.012195000000000001,"input_uncached_usd":0.0028350000000000003,"input_cached_usd":0.0,"output_usd":0.00936,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.847630667034537}
{"item_id":"senseval3.d000.s078.t005","gold_sense_keys":["fellow%1:18:03::"],"candidates":[{"index":1,"sense_key":"fellow%1:23:00::","synset_id":"mate.n.06"},{"index":2,"sense_key":"fellow%1:18:01::","synset_id":"boyfriend.n.01"},{"index":3,"sense_key":"fellow%1:18:03::","synset_id":"colleague.n.02"},{"index":4,"sense_key":"fellow%1:18:02::","synset_id":"companion.n.01"},{"index":5,"sense_key":"fellow%1:18:05::","synset_id":"fellow.n.05"},{"index":6,"sense_key":"fellow%1:18:00::","synset_id":"chap.n.01"},{"index":7,"sense_key":"fellow%1:18:04::","synset_id":"fellow.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fellow%1:18:03::","call_ids":["senseval3.d000.s078.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fellow%1:18:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":98},"cost":{"currency":"USD","total_usd":0.00636,"input_uncached_usd":0.0029700000000000004,"input_cached_usd":0.0,"output_usd":0.0033900000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8034988330909982}
{"item_id":"senseval3.d000.s078.t006","gold_sense_keys":["tenant%1:18:00::"],"candidates":[{"index":1,"sense_key":"tenant%1:18:02::","synset_id":"tenant.n.02"},{"index":2,"sense_key":"tenant%1:18:00::","synset_id":"tenant.n.01"},{"index":3,"sense_key":"tenant%1:18:01::","synset_id":"tenant.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tenant%1:18:00::","call_ids":["senseval3.d000.s078.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tenant%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":368,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.005050000000000001,"input_uncached_usd":0.00184,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0299846249399707}
{"item_id":"senseval3.d000.s078.t007","gold_sense_keys":["happen%2:30:00::"],"candidates":[{"index":1,"sense_key":"happen%2:30:00::","synset_id":"happen.v.01"},{"index":2,"sense_key":"happen%2:40:12::","synset_id":"find.v.01"},{"index":3,"sense_key":"happen%2:30:02::","synset_id":"happen.v.02"},{"index":4,"sense_key":"happen%2:30:01::","synset_id":"happen.v.04"},{"index":5,"sense_key":"happen%2:41:00::","synset_id":"happen.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"happen%2:30:00::","call_ids":["senseval3.d000.s078.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"happen%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":317,"reasoning_output_tokens":301},"cost":{"currency":"USD","total_usd":0.012055,"input_uncached_usd":0.0025450000000000004,"input_cached_usd":0.0,"output_usd":0.009510000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.505533457966521}
{"item_id":"senseval3.d000.s079.t001","gold_sense_keys":["die%2:30:00::"],"candidates":[{"index":1,"sense_key":"die%2:30:04::","synset_id":"fail.v.04"},{"index":2,"sense_key":"die%2:37:03::","synset_id":"die.v.05"},{"index":3,"sense_key":"die%2:37:01::","synset_id":"die.v.06"},{"index":4,"sense_key":"die%2:30:01::","synset_id":"die.v.10"},{"index":5,"sense_key":"die%2:30:05::","synset_id":"die.v.09"},{"index":6,"sense_key":"die%2:33:00::","synset_id":"die.v.08"},{"index":7,"sense_key":"die%2:30:00::","synset_id":"die.v.01"},{"index":8,"sense_key":"die%2:37:00::","synset_id":"die.v.03"},{"index":9,"sense_key":"die%2:35:00::","synset_id":"die.v.07"},{"index":10,"sense_key":"die%2:39:00::","synset_id":"die.v.02"},{"index":11,"sense_key":"die%2:30:02::","synset_id":"die.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"die%2:30:00::","call_ids":["senseval3.d000.s079.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"die%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.00537,"input_uncached_usd":0.00369,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4173031249083579}
{"item_id":"senseval3.d000.s080.t000","gold_sense_keys":["afternoon%1:28:00::"],"candidates":[{"index":1,"sense_key":"afternoon%1:28:00::","synset_id":"afternoon.n.01"},{"index":2,"sense_key":"afternoon%1:10:00::","synset_id":"good_afternoon.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"afternoon%1:28:00::","call_ids":["senseval3.d000.s080.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"afternoon%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":335,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.0034150000000000005,"input_uncached_usd":0.001675,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.361290334025398}
{"item_id":"senseval3.d000.s080.t001","gold_sense_keys":["seem%2:39:01::"],"candidates":[{"index":1,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":2,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seem%2:39:01::","call_ids":["senseval3.d000.s080.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seem%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":119},"cost":{"currency":"USD","total_usd":0.006275,"input_uncached_usd":0.002255,"input_cached_usd":0.0,"output_usd":0.00402,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.369110416038893}
{"item_id":"senseval3.d000.s080.t002","gold_sense_keys":["fall%2:38:03::"],"candidates":[{"index":1,"sense_key":"fall%2:41:15::","synset_id":"fall.v.17"},{"index":2,"sense_key":"fall%2:38:03::","synset_id":"fall.v.01"},{"index":3,"sense_key":"fall%2:41:00::","synset_id":"fall.v.13"},{"index":4,"sense_key":"fall%2:30:08::","synset_id":"fall.v.03"},{"index":5,"sense_key":"fall%2:30:13::","synset_id":"fall.v.32"},{"index":6,"sense_key":"fall%2:42:04::","synset_id":"fall.v.08"},{"index":7,"sense_key":"fall%2:30:15::","synset_id":"fall.v.31"},{"index":8,"sense_key":"fall%2:38:14::","synset_id":"fall.v.24"},{"index":9,"sense_key":"fall%2:35:00::","synset_id":"hang.v.05"},{"index":10,"sense_key":"fall%2:43:00::","synset_id":"precipitate.v.03"},{"index":11,"sense_key":"fall%2:40:09::","synset_id":"accrue.v.02"},{"index":12,"sense_key":"fall%2:40:05::","synset_id":"fall.v.09"},{"index":13,"sense_key":"fall%2:30:09::","synset_id":"fall.v.29"},{"index":14,"sense_key":"fall%2:42:03::","synset_id":"fall.v.04"},{"index":15,"sense_key":"fall%2:30:12::","synset_id":"fall.v.30"},{"index":16,"sense_key":"fall%2:30:06::","synset_id":"decrease.v.01"},{"index":17,"sense_key":"fall%2:38:00::","synset_id":"descend.v.01"},{"index":18,"sense_key":"fall%2:42:02::","synset_id":"fall.v.16"},{"index":19,"sense_key":"fall%2:40:12::","synset_id":"fall.v.18"},{"index":20,"sense_key":"fall%2:40:13::","synset_id":"fall.v.21"},{"index":21,"sense_key":"fall%2:30:00::","synset_id":"fall.v.10"},{"index":22,"sense_key":"fall%2:41:02::","synset_id":"fall.v.06"},{"index":23,"sense_key":"fall%2:38:15::","synset_id":"fall.v.23"},{"index":24,"sense_key":"fall%2:42:00::","synset_id":"fall.v.07"},{"index":25,"sense_key":"fall%2:30:11::","synset_id":"fall.v.26"},{"index":26,"sense_key":"fall%2:38:11::","synset_id":"fall.v.22"},{"index":27,"sense_key":"fall%2:38:01::","synset_id":"fall.v.15"},{"index":28,"sense_key":"fall%2:30:14::","synset_id":"fall.v.27"},{"index":29,"sense_key":"fall%2:40:03::","synset_id":"fall.v.14"},{"index":30,"sense_key":"fall%2:40:08::","synset_id":"fall.v.20"},{"index":31,"sense_key":"fall%2:30:10::","synset_id":"fall.v.28"},{"index":32,"sense_key":"fall%2:41:01::","synset_id":"fall.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fall%2:38:03::","call_ids":["senseval3.d000.s080.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fall%2:38:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1528,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":97},"cost":{"currency":"USD","total_usd":0.011030000000000002,"input_uncached_usd":0.007640000000000001,"input_cached_usd":0.0,"output_usd":0.0033900000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9551628329791129}
{"item_id":"senseval3.d000.s080.t003","gold_sense_keys":["roof%1:06:00::"],"candidates":[{"index":1,"sense_key":"roof%1:17:00::","synset_id":"roof.n.03"},{"index":2,"sense_key":"roof%1:06:01::","synset_id":"roof.n.02"},{"index":3,"sense_key":"roof%1:10:00::","synset_id":"ceiling.n.03"},{"index":4,"sense_key":"roof%1:06:00::","synset_id":"roof.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"roof%1:06:00::","call_ids":["senseval3.d000.s080.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"roof%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.00396,"input_uncached_usd":0.0021300000000000004,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8510844999691471}
{"item_id":"senseval3.d000.s080.t004","gold_sense_keys":["obscure%5:00:00:unclear:00"],"candidates":[{"index":1,"sense_key":"obscure%5:00:00:incomprehensible:00","synset_id":"dark.s.06"},{"index":2,"sense_key":"obscure%5:00:00:unconnected:00","synset_id":"apart.s.01"},{"index":3,"sense_key":"obscure%5:00:00:inglorious:00","synset_id":"obscure.s.04"},{"index":4,"sense_key":"obscure%5:00:00:concealed:00","synset_id":"hidden.s.03"},{"index":5,"sense_key":"obscure%5:00:00:unclear:00","synset_id":"obscure.s.01"},{"index":6,"sense_key":"obscure%5:00:00:inconspicuous:00","synset_id":"obscure.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"obscure%5:00:00:unclear:00","call_ids":["senseval3.d000.s080.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"obscure%5:00:00:unclear:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":608,"cached_input_tokens":0,"output_tokens":170,"reasoning_output_tokens":155},"cost":{"currency":"USD","total_usd":0.008140000000000001,"input_uncached_usd":0.00304,"input_cached_usd":0.0,"output_usd":0.0051,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0033422498963773}
{"item_id":"senseval3.d000.s082.t001","gold_sense_keys":["witness%2:39:00::"],"candidates":[{"index":1,"sense_key":"witness%2:39:01::","synset_id":"witness.v.02"},{"index":2,"sense_key":"witness%2:39:00::","synset_id":"witness.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"witness%2:39:00::","call_ids":["senseval3.d000.s082.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"witness%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":347,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.004195,"input_uncached_usd":0.0017350000000000002,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.845153999980539}
{"item_id":"senseval3.d000.s082.t002","gold_sense_keys":["fall%1:11:00::"],"candidates":[{"index":1,"sense_key":"fall%1:04:02::","synset_id":"capitulation.n.03"},{"index":2,"sense_key":"fall%1:11:03::","synset_id":"fall.n.03"},{"index":3,"sense_key":"fall%1:28:00::","synset_id":"fall.n.01"},{"index":4,"sense_key":"fall%1:07:00::","synset_id":"drop.n.03"},{"index":5,"sense_key":"fall%1:17:00::","synset_id":"descent.n.05"},{"index":6,"sense_key":"fall%1:04:01::","synset_id":"spill.n.04"},{"index":7,"sense_key":"fall%1:04:03::","synset_id":"fall.n.05"},{"index":8,"sense_key":"fall%1:11:01::","synset_id":"fall.n.06"},{"index":9,"sense_key":"fall%1:11:04::","synset_id":"fall.n.07"},{"index":10,"sense_key":"fall%1:28:01::","synset_id":"twilight.n.01"},{"index":11,"sense_key":"fall%1:11:02::","synset_id":"fall.n.10"},{"index":12,"sense_key":"fall%1:11:00::","synset_id":"drop.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"fall%1:11:00::","call_ids":["senseval3.d000.s082.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"fall%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":169,"reasoning_output_tokens":153},"cost":{"currency":"USD","total_usd":0.008925,"input_uncached_usd":0.0038550000000000004,"input_cached_usd":0.0,"output_usd":0.00507,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1835202910006046}
{"item_id":"senseval3.d000.s082.t004","gold_sense_keys":["impact%1:11:00::"],"candidates":[{"index":1,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"},{"index":2,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":3,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":4,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"impact%1:11:00::","call_ids":["senseval3.d000.s082.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"impact%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":423,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.003585,"input_uncached_usd":0.002115,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3857626660028473}
{"item_id":"senseval3.d000.s082.t005","gold_sense_keys":["body%1:08:00::"],"candidates":[{"index":1,"sense_key":"body%1:10:00::","synset_id":"body.n.08"},{"index":2,"sense_key":"body%1:06:00::","synset_id":"body.n.11"},{"index":3,"sense_key":"body%1:08:00::","synset_id":"body.n.01"},{"index":4,"sense_key":"body%1:08:02::","synset_id":"body.n.03"},{"index":5,"sense_key":"body%1:06:01::","synset_id":"soundbox.n.01"},{"index":6,"sense_key":"body%1:07:00::","synset_id":"consistency.n.01"},{"index":7,"sense_key":"body%1:14:00::","synset_id":"body.n.02"},{"index":8,"sense_key":"body%1:07:01::","synset_id":"body.n.09"},{"index":9,"sense_key":"body%1:17:00::","synset_id":"body.n.04"},{"index":10,"sense_key":"body%1:08:01::","synset_id":"torso.n.01"},{"index":11,"sense_key":"body%1:14:02::","synset_id":"body.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"body%1:08:00::","call_ids":["senseval3.d000.s082.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"body%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":712,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0194,"input_uncached_usd":0.0035600000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.230692417011596}
{"item_id":"senseval3.d000.s082.t006","gold_sense_keys":["smash%2:35:05::"],"candidates":[{"index":1,"sense_key":"smash%2:35:00::","synset_id":"smash.v.01"},{"index":2,"sense_key":"smash%2:35:08::","synset_id":"bang_up.v.01"},{"index":3,"sense_key":"smash%2:30:00::","synset_id":"smash.v.02"},{"index":4,"sense_key":"smash%2:35:05::","synset_id":"smash.v.08"},{"index":5,"sense_key":"smash%2:30:10::","synset_id":"smash.v.10"},{"index":6,"sense_key":"smash%2:35:06::","synset_id":"smash.v.07"},{"index":7,"sense_key":"smash%2:37:00::","synset_id":"crush.v.05"},{"index":8,"sense_key":"smash%2:33:00::","synset_id":"smash.v.09"},{"index":9,"sense_key":"smash%2:38:02::","synset_id":"smash.v.04"},{"index":10,"sense_key":"smash%2:40:00::","synset_id":"bankrupt.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"smash%2:35:05::","call_ids":["senseval3.d000.s082.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"smash%2:35:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":160,"reasoning_output_tokens":144},"cost":{"currency":"USD","total_usd":0.008110000000000001,"input_uncached_usd":0.0033100000000000004,"input_cached_usd":0.0,"output_usd":0.0048000000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1613052500179037}
{"item_id":"senseval3.d000.s082.t007","gold_sense_keys":["pavement%1:06:00::","pavement%1:06:01::"],"candidates":[{"index":1,"sense_key":"pavement%1:27:00::","synset_id":"paving.n.01"},{"index":2,"sense_key":"pavement%1:06:00::","synset_id":"pavement.n.01"},{"index":3,"sense_key":"pavement%1:06:01::","synset_id":"sidewalk.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pavement%1:06:00::","call_ids":["senseval3.d000.s082.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pavement%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":363,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017655,"input_uncached_usd":0.0018150000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.569254750036635}
{"item_id":"senseval3.d000.s082.t008","gold_sense_keys":["basement%1:06:00::"],"candidates":[{"index":1,"sense_key":"basement%1:06:01::","synset_id":"basement.n.02"},{"index":2,"sense_key":"basement%1:06:00::","synset_id":"basement.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"basement%1:06:00::","call_ids":["senseval3.d000.s082.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"basement%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":333,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.003225,"input_uncached_usd":0.001665,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.046037583029829}
{"item_id":"senseval3.d000.s082.t009","gold_sense_keys":["delivery%1:04:04::"],"candidates":[{"index":1,"sense_key":"delivery%1:04:03::","synset_id":"pitch.n.02"},{"index":2,"sense_key":"delivery%1:04:00::","synset_id":"rescue.n.01"},{"index":3,"sense_key":"delivery%1:04:04::","synset_id":"delivery.n.01"},{"index":4,"sense_key":"delivery%1:10:00::","synset_id":"manner_of_speaking.n.01"},{"index":5,"sense_key":"delivery%1:04:02::","synset_id":"delivery.n.07"},{"index":6,"sense_key":"delivery%1:11:00::","synset_id":"delivery.n.02"},{"index":7,"sense_key":"delivery%1:04:01::","synset_id":"delivery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"delivery%1:04:04::","call_ids":["senseval3.d000.s082.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"delivery%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.005325,"input_uncached_usd":0.002745,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7751212500734255}
{"item_id":"senseval3.d000.s082.t010","gold_sense_keys":["entrance%1:06:00::"],"candidates":[{"index":1,"sense_key":"entrance%1:04:00::","synset_id":"entrance.n.03"},{"index":2,"sense_key":"entrance%1:11:00::","synset_id":"entrance.n.02"},{"index":3,"sense_key":"entrance%1:06:00::","synset_id":"entrance.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"entrance%1:06:00::","call_ids":["senseval3.d000.s082.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"entrance%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":397,"cached_input_tokens":0,"output_tokens":41,"reasoning_output_tokens":25},"cost":{"currency":"USD","total_usd":0.003215,"input_uncached_usd":0.0019850000000000002,"input_cached_usd":0.0,"output_usd":0.00123,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.824095125077292}
{"item_id":"senseval3.d000.s083.t002","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"},{"index":3,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":4,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d000.s083.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0182,"input_uncached_usd":0.00236,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":18.03168275009375}
{"item_id":"senseval3.d000.s083.t005","gold_sense_keys":["feel%2:39:00::"],"candidates":[{"index":1,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":2,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":3,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":4,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":5,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":6,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":7,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":8,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"},{"index":9,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":10,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":11,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":12,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":13,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"feel%2:39:00::","call_ids":["senseval3.d000.s083.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"feel%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":816,"cached_input_tokens":0,"output_tokens":348,"reasoning_output_tokens":332},"cost":{"currency":"USD","total_usd":0.01452,"input_uncached_usd":0.00408,"input_cached_usd":0.0,"output_usd":0.01044,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.785056583001278}
{"item_id":"senseval3.d000.s083.t006","gold_sense_keys":["blood%1:08:00::"],"candidates":[{"index":1,"sense_key":"blood%1:07:00::","synset_id":"blood.n.02"},{"index":2,"sense_key":"blood%1:14:01::","synset_id":"blood.n.05"},{"index":3,"sense_key":"blood%1:18:00::","synset_id":"rake.n.01"},{"index":4,"sense_key":"blood%1:08:00::","synset_id":"blood.n.01"},{"index":5,"sense_key":"blood%1:14:00::","synset_id":"lineage.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"blood%1:08:00::","call_ids":["senseval3.d000.s083.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"blood%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.004745,"input_uncached_usd":0.002465,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3809505000244826}
{"item_id":"senseval3.d000.s083.t007","gold_sense_keys":["drain%2:38:00::"],"candidates":[{"index":1,"sense_key":"drain%2:38:00::","synset_id":"drain.v.01"},{"index":2,"sense_key":"drain%2:34:00::","synset_id":"drain.v.02"},{"index":3,"sense_key":"drain%2:30:01::","synset_id":"drain.v.03"},{"index":4,"sense_key":"drain%2:30:04::","synset_id":"enfeeble.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drain%2:38:00::","call_ids":["senseval3.d000.s083.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drain%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":413,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.005275,"input_uncached_usd":0.002065,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.404256624984555}
{"item_id":"senseval3.d000.s083.t008","gold_sense_keys":["cheek%1:08:00::"],"candidates":[{"index":1,"sense_key":"cheek%1:08:00::","synset_id":"cheek.n.01"},{"index":2,"sense_key":"cheek%1:07:00::","synset_id":"boldness.n.02"},{"index":3,"sense_key":"cheek%1:10:00::","synset_id":"impudence.n.01"},{"index":4,"sense_key":"cheek%1:08:01::","synset_id":"buttock.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cheek%1:08:00::","call_ids":["senseval3.d000.s083.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cheek%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":42,"reasoning_output_tokens":27},"cost":{"currency":"USD","total_usd":0.0033700000000000006,"input_uncached_usd":0.0021100000000000003,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7472438750555739}
{"item_id":"senseval3.d000.s084.t000","gold_sense_keys":["mutter%2:32:01::"],"candidates":[{"index":1,"sense_key":"mutter%2:32:01::","synset_id":"mumble.v.01"},{"index":2,"sense_key":"mutter%2:32:00::","synset_id":"murmur.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mutter%2:32:01::","call_ids":["senseval3.d000.s084.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mutter%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":358,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.00383,"input_uncached_usd":0.0017900000000000001,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0058290000306442}
{"item_id":"senseval3.d000.s084.t001","gold_sense_keys":["terrible%5:00:00:alarming:00"],"candidates":[{"index":1,"sense_key":"terrible%5:00:00:intense:00","synset_id":"severe.s.01"},{"index":2,"sense_key":"terrible%5:00:01:bad:00","synset_id":"atrocious.s.02"},{"index":3,"sense_key":"terrible%5:00:00:extraordinary:00","synset_id":"frightful.s.02"},{"index":4,"sense_key":"terrible%5:00:00:alarming:00","synset_id":"awful.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"terrible%5:00:00:alarming:00","call_ids":["senseval3.d000.s084.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"terrible%5:00:00:alarming:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":1510,"reasoning_output_tokens":1482},"cost":{"currency":"USD","total_usd":0.047755,"input_uncached_usd":0.002455,"input_cached_usd":0.0,"output_usd":0.0453,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":30.73817170795519}
{"item_id":"senseval3.d000.s084.t002","gold_sense_keys":["walk%2:38:00::"],"candidates":[{"index":1,"sense_key":"walk%2:33:00::","synset_id":"walk.v.03"},{"index":2,"sense_key":"walk%2:41:01::","synset_id":"walk.v.07"},{"index":3,"sense_key":"walk%2:33:01::","synset_id":"walk.v.05"},{"index":4,"sense_key":"walk%2:38:04::","synset_id":"walk.v.10"},{"index":5,"sense_key":"walk%2:38:02::","synset_id":"walk.v.08"},{"index":6,"sense_key":"walk%2:38:01::","synset_id":"walk.v.02"},{"index":7,"sense_key":"walk%2:41:00::","synset_id":"walk.v.06"},{"index":8,"sense_key":"walk%2:38:05::","synset_id":"walk.v.09"},{"index":9,"sense_key":"walk%2:38:00::","synset_id":"walk.v.01"},{"index":10,"sense_key":"walk%2:38:03::","synset_id":"walk.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"walk%2:38:00::","call_ids":["senseval3.d000.s084.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"walk%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":363,"reasoning_output_tokens":347},"cost":{"currency":"USD","total_usd":0.013940000000000001,"input_uncached_usd":0.00305,"input_cached_usd":0.0,"output_usd":0.01089,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.11737200000789}
{"item_id":"senseval3.d000.s084.t003","gold_sense_keys":["deliberate%5:00:00:unhurried:00"],"candidates":[{"index":1,"sense_key":"deliberate%5:00:00:unhurried:00","synset_id":"careful.s.02"},{"index":2,"sense_key":"deliberate%5:00:00:intended:00","synset_id":"deliberate.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"deliberate%5:00:00:unhurried:00","call_ids":["senseval3.d000.s084.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"deliberate%5:00:00:unhurried:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":367,"cached_input_tokens":0,"output_tokens":319,"reasoning_output_tokens":304},"cost":{"currency":"USD","total_usd":0.011405,"input_uncached_usd":0.001835,"input_cached_usd":0.0,"output_usd":0.00957,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.10150466603227}
{"item_id":"senseval3.d000.s084.t004","gold_sense_keys":["slowness%1:07:00::"],"candidates":[{"index":1,"sense_key":"slowness%1:09:01::","synset_id":"retardation.n.04"},{"index":2,"sense_key":"slowness%1:09:02::","synset_id":"awkwardness.n.01"},{"index":3,"sense_key":"slowness%1:07:00::","synset_id":"slowness.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"slowness%1:07:00::","call_ids":["senseval3.d000.s084.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"slowness%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":406,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.00362,"input_uncached_usd":0.00203,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3288679170655087}
{"item_id":"senseval3.d000.s084.t005","gold_sense_keys":["elevator%1:06:00::"],"candidates":[{"index":1,"sense_key":"elevator%1:06:01::","synset_id":"elevator.n.02"},{"index":2,"sense_key":"elevator%1:06:00::","synset_id":"elevator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"elevator%1:06:00::","call_ids":["senseval3.d000.s084.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"elevator%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":353,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.003535,"input_uncached_usd":0.001765,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6305516669526696}
{"item_id":"senseval3.d000.s085.t001","gold_sense_keys":["pour%2:38:04::"],"candidates":[{"index":1,"sense_key":"pour%2:38:04::","synset_id":"decant.v.01"},{"index":2,"sense_key":"pour%2:40:00::","synset_id":"pour.v.05"},{"index":3,"sense_key":"pour%2:38:02::","synset_id":"pour.v.04"},{"index":4,"sense_key":"pour%2:38:03::","synset_id":"pour.v.01"},{"index":5,"sense_key":"pour%2:43:00::","synset_id":"pour.v.06"},{"index":6,"sense_key":"pour%2:38:00::","synset_id":"pour.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pour%2:38:04::","call_ids":["senseval3.d000.s085.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pour%2:38:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018405,"input_uncached_usd":0.002565,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.534207041957416}
{"item_id":"senseval3.d000.s085.t002","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"},{"index":2,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":3,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":4,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":5,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s085.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.004385,"input_uncached_usd":0.0024950000000000003,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3634839170845225}
{"item_id":"senseval3.d000.s085.t004","gold_sense_keys":["hand%1:08:00::"],"candidates":[{"index":1,"sense_key":"hand%1:18:01::","synset_id":"bridge_player.n.01"},{"index":2,"sense_key":"hand%1:05:01::","synset_id":"hand.n.13"},{"index":3,"sense_key":"hand%1:06:00::","synset_id":"hand.n.08"},{"index":4,"sense_key":"hand%1:18:00::","synset_id":"hired_hand.n.01"},{"index":5,"sense_key":"hand%1:04:00::","synset_id":"hand.n.14"},{"index":6,"sense_key":"hand%1:10:01::","synset_id":"hand.n.12"},{"index":7,"sense_key":"hand%1:23:00::","synset_id":"hand.n.09"},{"index":8,"sense_key":"hand%1:14:00::","synset_id":"hand.n.06"},{"index":9,"sense_key":"hand%1:15:00::","synset_id":"hand.n.05"},{"index":10,"sense_key":"hand%1:10:00::","synset_id":"handwriting.n.01"},{"index":11,"sense_key":"hand%1:09:01::","synset_id":"hand.n.07"},{"index":12,"sense_key":"hand%1:18:02::","synset_id":"hand.n.10"},{"index":13,"sense_key":"hand%1:09:00::","synset_id":"hand.n.04"},{"index":14,"sense_key":"hand%1:08:00::","synset_id":"hand.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"hand%1:08:00::","call_ids":["senseval3.d000.s085.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"hand%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":872,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.00679,"input_uncached_usd":0.00436,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1709419999970123}
{"item_id":"senseval3.d000.s085.t005","gold_sense_keys":["flop%2:38:01::"],"candidates":[{"index":1,"sense_key":"flop%2:41:00::","synset_id":"fall_through.v.01"},{"index":2,"sense_key":"flop%2:38:01::","synset_id":"flop.v.01"},{"index":3,"sense_key":"flop%2:38:00::","synset_id":"flop.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"flop%2:38:01::","call_ids":["senseval3.d000.s085.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"flop%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":375,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.004515,"input_uncached_usd":0.0018750000000000001,"input_cached_usd":0.0,"output_usd":0.00264,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7692286249948665}
{"item_id":"senseval3.d000.s085.t006","gold_sense_keys":["chair%1:06:00::"],"candidates":[{"index":1,"sense_key":"chair%1:06:00::","synset_id":"chair.n.01"},{"index":2,"sense_key":"chair%1:06:01::","synset_id":"electric_chair.n.01"},{"index":3,"sense_key":"chair%1:06:02::","synset_id":"chair.n.05"},{"index":4,"sense_key":"chair%1:04:00::","synset_id":"professorship.n.01"},{"index":5,"sense_key":"chair%1:18:00::","synset_id":"president.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"chair%1:06:00::","call_ids":["senseval3.d000.s085.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"chair%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.0040999999999999995,"input_uncached_usd":0.00251,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.493663875036873}
{"item_id":"senseval3.d000.s086.t002","gold_sense_keys":["feel%2:29:00::","feel%2:37:00::"],"candidates":[{"index":1,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":2,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":3,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":4,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":7,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":8,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":9,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":10,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":11,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":12,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":13,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"feel%2:29:00::","call_ids":["senseval3.d000.s086.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"feel%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":815,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019915000000000002,"input_uncached_usd":0.0040750000000000005,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.072875750018284}
{"item_id":"senseval3.d000.s086.t003","gold_sense_keys":["bother%2:41:00::"],"candidates":[{"index":1,"sense_key":"bother%2:41:00::","synset_id":"trouble_oneself.v.01"},{"index":2,"sense_key":"bother%2:37:03::","synset_id":"bother.v.05"},{"index":3,"sense_key":"bother%2:41:01::","synset_id":"trouble.v.02"},{"index":4,"sense_key":"bother%2:38:00::","synset_id":"bother.v.04"},{"index":5,"sense_key":"bother%2:37:02::","synset_id":"bother.v.06"},{"index":6,"sense_key":"bother%2:37:00::","synset_id":"annoy.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bother%2:41:00::","call_ids":["senseval3.d000.s086.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bother%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.004985,"input_uncached_usd":0.002765,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9665425840066746}
{"item_id":"senseval3.d000.s086.t004","gold_sense_keys":["ask%2:32:02::"],"candidates":[{"index":1,"sense_key":"ask%2:32:01::","synset_id":"ask.v.02"},{"index":2,"sense_key":"ask%2:32:05::","synset_id":"ask.v.04"},{"index":3,"sense_key":"ask%2:42:00::","synset_id":"necessitate.v.01"},{"index":4,"sense_key":"ask%2:32:04::","synset_id":"ask.v.05"},{"index":5,"sense_key":"ask%2:32:09::","synset_id":"ask.v.07"},{"index":6,"sense_key":"ask%2:32:02::","synset_id":"ask.v.03"},{"index":7,"sense_key":"ask%2:32:00::","synset_id":"ask.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"ask%2:32:02::","call_ids":["senseval3.d000.s086.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"ask%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":138},"cost":{"currency":"USD","total_usd":0.007530000000000001,"input_uncached_usd":0.0029100000000000003,"input_cached_usd":0.0,"output_usd":0.00462,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":171.37482849997468}
{"item_id":"senseval3.d000.s086.t005","gold_sense_keys":["question%1:10:03::"],"candidates":[{"index":1,"sense_key":"question%1:10:03::","synset_id":"question.n.03"},{"index":2,"sense_key":"question%1:10:00::","synset_id":"question.n.01"},{"index":3,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"},{"index":4,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":5,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"},{"index":6,"sense_key":"question%1:10:04::","synset_id":"question.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"question%1:10:03::","call_ids":["senseval3.d000.s086.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"question%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":181,"reasoning_output_tokens":165},"cost":{"currency":"USD","total_usd":0.008245,"input_uncached_usd":0.0028150000000000002,"input_cached_usd":0.0,"output_usd":0.00543,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.077506250003353}
{"item_id":"senseval3.d000.s088.t002","gold_sense_keys":["coincidence%1:11:00::"],"candidates":[{"index":1,"sense_key":"coincidence%1:07:01::","synset_id":"concurrence.n.04"},{"index":2,"sense_key":"coincidence%1:11:00::","synset_id":"coincidence.n.01"},{"index":3,"sense_key":"coincidence%1:07:02::","synset_id":"coincidence.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"coincidence%1:11:00::","call_ids":["senseval3.d000.s088.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"coincidence%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":405,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004035,"input_uncached_usd":0.0020250000000000003,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7184052499942482}
{"item_id":"senseval3.d000.s089.t000","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"},{"index":2,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s089.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":329,"cached_input_tokens":0,"output_tokens":940,"reasoning_output_tokens":918},"cost":{"currency":"USD","total_usd":0.029845000000000003,"input_uncached_usd":0.0016450000000000002,"input_cached_usd":0.0,"output_usd":0.0282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":17.55644791701343}
{"item_id":"senseval3.d000.s090.t001","gold_sense_keys":["happy%3:00:00::"],"candidates":[{"index":1,"sense_key":"happy%5:00:00:felicitous:00","synset_id":"happy.s.03"},{"index":2,"sense_key":"happy%5:00:00:fortunate:00","synset_id":"felicitous.s.01"},{"index":3,"sense_key":"happy%5:00:00:willing:00","synset_id":"glad.s.01"},{"index":4,"sense_key":"happy%3:00:00::","synset_id":"happy.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"happy%3:00:00::","call_ids":["senseval3.d000.s090.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"happy%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":97},"cost":{"currency":"USD","total_usd":0.005575,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.0033900000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0451614999910817}
{"item_id":"senseval3.d000.s090.t002","gold_sense_keys":["lousy%5:00:00:nasty:00"],"candidates":[{"index":1,"sense_key":"lousy%5:00:00:dirty:01","synset_id":"lousy.s.02"},{"index":2,"sense_key":"lousy%5:00:00:nasty:00","synset_id":"dirty.s.01"},{"index":3,"sense_key":"lousy%5:00:00:bad:00","synset_id":"icky.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"lousy%5:00:00:nasty:00","call_ids":["senseval3.d000.s090.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"lousy%5:00:00:nasty:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":394,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01781,"input_uncached_usd":0.00197,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.30421450000722}
{"item_id":"senseval3.d000.s091.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":2,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":3,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":8,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":9,"sense_key":"man%1:05:00::","synset_id":"world.n.08"},{"index":10,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":11,"sense_key":"man%1:18:07::","synset_id":"man.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s091.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":799,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":108},"cost":{"currency":"USD","total_usd":0.0077150000000000005,"input_uncached_usd":0.003995,"input_cached_usd":0.0,"output_usd":0.00372,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.676100124954246}
{"item_id":"senseval3.d000.s091.t001","gold_sense_keys":["have%2:40:00::"],"candidates":[{"index":1,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":2,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":3,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":4,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":5,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":6,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":7,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":8,"sense_key":"have%2:35:00::","synset_id":"take.v.35"},{"index":9,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":10,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":11,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":14,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":15,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":16,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":17,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":18,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":19,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":18,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d000.s091.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":18,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1098,"cached_input_tokens":0,"output_tokens":342,"reasoning_output_tokens":327},"cost":{"currency":"USD","total_usd":0.01575,"input_uncached_usd":0.00549,"input_cached_usd":0.0,"output_usd":0.01026,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.068545707967132}
{"item_id":"senseval3.d000.s091.t002","gold_sense_keys":["family%1:14:00::","family%1:14:02::"],"candidates":[{"index":1,"sense_key":"family%1:18:00::","synset_id":"kin.n.01"},{"index":2,"sense_key":"family%1:14:00::","synset_id":"family.n.02"},{"index":3,"sense_key":"family%1:14:06::","synset_id":"syndicate.n.01"},{"index":4,"sense_key":"family%1:14:05::","synset_id":"family.n.08"},{"index":5,"sense_key":"family%1:14:02::","synset_id":"family.n.01"},{"index":6,"sense_key":"family%1:14:03::","synset_id":"family.n.06"},{"index":7,"sense_key":"family%1:14:01::","synset_id":"family.n.04"},{"index":8,"sense_key":"family%1:14:04::","synset_id":"class.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"family%1:18:00::","call_ids":["senseval3.d000.s091.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"family%1:18:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":2600,"reasoning_output_tokens":2558},"cost":{"currency":"USD","total_usd":0.081145,"input_uncached_usd":0.0031450000000000002,"input_cached_usd":0.0,"output_usd":0.078,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":46.65473354200367}
{"item_id":"senseval3.d000.s091.t003","gold_sense_keys":["suffer%2:29:00::","suffer%2:37:00::"],"candidates":[{"index":1,"sense_key":"suffer%2:31:00::","synset_id":"digest.v.03"},{"index":2,"sense_key":"suffer%2:37:00::","synset_id":"suffer.v.03"},{"index":3,"sense_key":"suffer%2:39:00::","synset_id":"hurt.v.06"},{"index":4,"sense_key":"suffer%2:30:02::","synset_id":"suffer.v.11"},{"index":5,"sense_key":"suffer%2:29:00::","synset_id":"suffer.v.06"},{"index":6,"sense_key":"suffer%2:30:00::","synset_id":"suffer.v.05"},{"index":7,"sense_key":"suffer%2:42:04::","synset_id":"suffer.v.09"},{"index":8,"sense_key":"suffer%2:39:01::","synset_id":"suffer.v.01"},{"index":9,"sense_key":"suffer%2:29:03::","synset_id":"suffer.v.08"},{"index":10,"sense_key":"suffer%2:39:02::","synset_id":"suffer.v.10"},{"index":11,"sense_key":"suffer%2:29:01::","synset_id":"suffer.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"suffer%2:37:00::","call_ids":["senseval3.d000.s091.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"suffer%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":675,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":119},"cost":{"currency":"USD","total_usd":0.007425000000000001,"input_uncached_usd":0.0033750000000000004,"input_cached_usd":0.0,"output_usd":0.00405,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5414446250069886}
{"item_id":"senseval3.d000.s092.t000","gold_sense_keys":["say%2:32:00::"],"candidates":[{"index":1,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":2,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":3,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":6,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":9,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":10,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":11,"sense_key":"say%2:32:08::","synset_id":"say.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval3.d000.s092.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":734,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":133},"cost":{"currency":"USD","total_usd":0.008140000000000001,"input_uncached_usd":0.00367,"input_cached_usd":0.0,"output_usd":0.00447,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.040770416962914}
{"item_id":"senseval3.d000.s092.t001","gold_sense_keys":["kill%2:35:00::"],"candidates":[{"index":1,"sense_key":"kill%2:35:02::","synset_id":"kill.v.09"},{"index":2,"sense_key":"kill%2:39:00::","synset_id":"kill.v.05"},{"index":3,"sense_key":"kill%2:35:05::","synset_id":"kill.v.08"},{"index":4,"sense_key":"kill%2:35:01::","synset_id":"kill.v.10"},{"index":5,"sense_key":"kill%2:42:00::","synset_id":"kill.v.04"},{"index":6,"sense_key":"kill%2:30:00::","synset_id":"kill.v.12"},{"index":7,"sense_key":"kill%2:30:03::","synset_id":"kill.v.14"},{"index":8,"sense_key":"kill%2:35:06::","synset_id":"kill.v.07"},{"index":9,"sense_key":"kill%2:41:01::","synset_id":"kill.v.02"},{"index":10,"sense_key":"kill%2:35:00::","synset_id":"kill.v.01"},{"index":11,"sense_key":"kill%2:30:02::","synset_id":"kill.v.15"},{"index":12,"sense_key":"kill%2:34:00::","synset_id":"toss_off.v.02"},{"index":13,"sense_key":"kill%2:37:00::","synset_id":"kill.v.06"},{"index":14,"sense_key":"kill%2:30:08::","synset_id":"stamp_out.v.01"},{"index":15,"sense_key":"kill%2:30:04::","synset_id":"kill.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"kill%2:35:00::","call_ids":["senseval3.d000.s092.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"kill%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":124},"cost":{"currency":"USD","total_usd":0.008335000000000002,"input_uncached_usd":0.004135000000000001,"input_cached_usd":0.0,"output_usd":0.0042,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.749380790977739}
{"item_id":"senseval3.d000.s092.t002","gold_sense_keys":["man%1:18:00::","man%1:18:04::"],"candidates":[{"index":1,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":2,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":3,"sense_key":"man%1:05:00::","synset_id":"world.n.08"},{"index":4,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":5,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":6,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":7,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":8,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":9,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":10,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":11,"sense_key":"man%1:06:00::","synset_id":"man.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s092.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":788,"cached_input_tokens":0,"output_tokens":255,"reasoning_output_tokens":239},"cost":{"currency":"USD","total_usd":0.01159,"input_uncached_usd":0.00394,"input_cached_usd":0.0,"output_usd":0.0076500000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.303177583031356}
{"item_id":"senseval3.d000.s092.t003","gold_sense_keys":["favor%1:04:00::"],"candidates":[{"index":1,"sense_key":"favor%1:12:00::","synset_id":"favor.n.04"},{"index":2,"sense_key":"favor%1:04:00::","synset_id":"favor.n.01"},{"index":3,"sense_key":"favor%1:07:00::","synset_id":"favor.n.02"},{"index":4,"sense_key":"favor%1:06:00::","synset_id":"party_favor.n.01"},{"index":5,"sense_key":"favor%1:09:00::","synset_id":"favor.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"favor%1:04:00::","call_ids":["senseval3.d000.s092.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"favor%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004125,"input_uncached_usd":0.002085,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.888431250001304}
{"item_id":"senseval3.d000.s093.t000","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":2,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":3,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":4,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":5,"sense_key":"think%2:30:00::","synset_id":"think.v.13"},{"index":6,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":7,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":8,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":9,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":10,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":11,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":12,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":13,"sense_key":"think%2:31:13::","synset_id":"think.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval3.d000.s093.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":830,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01999,"input_uncached_usd":0.00415,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.834953999961726}
{"item_id":"senseval3.d000.s093.t001","gold_sense_keys":["mean%2:32:01::"],"candidates":[{"index":1,"sense_key":"mean%2:31:01::","synset_id":"think_of.v.04"},{"index":2,"sense_key":"mean%2:42:03::","synset_id":"mean.v.05"},{"index":3,"sense_key":"mean%2:32:01::","synset_id":"mean.v.01"},{"index":4,"sense_key":"mean%2:31:00::","synset_id":"intend.v.01"},{"index":5,"sense_key":"mean%2:31:05::","synset_id":"mean.v.07"},{"index":6,"sense_key":"mean%2:32:03::","synset_id":"mean.v.03"},{"index":7,"sense_key":"mean%2:42:00::","synset_id":"entail.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"mean%2:32:01::","call_ids":["senseval3.d000.s093.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"mean%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01871,"input_uncached_usd":0.00287,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.445191707927734}
{"item_id":"senseval3.d000.s093.t002","gold_sense_keys":["say%2:32:00::"],"candidates":[{"index":1,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":2,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":3,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":4,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":5,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":6,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":7,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":8,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":9,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":10,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":11,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval3.d000.s093.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":744,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":140},"cost":{"currency":"USD","total_usd":0.008400000000000001,"input_uncached_usd":0.00372,"input_cached_usd":0.0,"output_usd":0.00468,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.264655375038274}
{"item_id":"senseval3.d000.s094.t000","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s094.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":332,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0175,"input_uncached_usd":0.0016600000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.055674709030427}
{"item_id":"senseval3.d000.s094.t001","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":2,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"},{"index":5,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s094.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":466,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.00383,"input_uncached_usd":0.00233,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2577537919860333}
{"item_id":"senseval3.d000.s094.t002","gold_sense_keys":["large%3:00:00::"],"candidates":[{"index":1,"sense_key":"large%5:00:00:pregnant:00","synset_id":"big.s.12"},{"index":2,"sense_key":"large%3:00:00::","synset_id":"large.a.01"},{"index":3,"sense_key":"large%5:00:00:significant:00","synset_id":"large.s.01"},{"index":4,"sense_key":"large%5:00:00:rhetorical:00","synset_id":"bombastic.s.01"},{"index":5,"sense_key":"large%5:00:00:generous:02","synset_id":"big.s.10"},{"index":6,"sense_key":"large%5:00:00:comprehensive:00","synset_id":"large.s.05"},{"index":7,"sense_key":"large%5:00:00:conspicuous:00","synset_id":"big.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"large%3:00:00::","call_ids":["senseval3.d000.s094.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"large%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.00524,"input_uncached_usd":0.00305,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6966506669996306}
{"item_id":"senseval3.d000.s094.t003","gold_sense_keys":["sad%3:00:00::"],"candidates":[{"index":1,"sense_key":"sad%3:00:00::","synset_id":"sad.a.01"},{"index":2,"sense_key":"sad%5:00:00:bad:00","synset_id":"deplorable.s.01"},{"index":3,"sense_key":"sad%5:00:00:sorrowful:00","synset_id":"sad.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sad%3:00:00::","call_ids":["senseval3.d000.s094.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sad%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.0042699999999999995,"input_uncached_usd":0.00217,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.505487292073667}
{"item_id":"senseval3.d000.s094.t004","gold_sense_keys":["hurt%2:37:00::"],"candidates":[{"index":1,"sense_key":"hurt%2:30:04::","synset_id":"hurt.v.04"},{"index":2,"sense_key":"hurt%2:39:00::","synset_id":"ache.v.03"},{"index":3,"sense_key":"hurt%2:39:01::","synset_id":"hurt.v.06"},{"index":4,"sense_key":"hurt%2:29:02::","synset_id":"suffer.v.06"},{"index":5,"sense_key":"hurt%2:29:01::","synset_id":"hurt.v.02"},{"index":6,"sense_key":"hurt%2:37:00::","synset_id":"hurt.v.05"},{"index":7,"sense_key":"hurt%2:37:01::","synset_id":"pain.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"hurt%2:37:00::","call_ids":["senseval3.d000.s094.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"hurt%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.004815,"input_uncached_usd":0.0026550000000000002,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6652429579989985}
{"item_id":"senseval3.d000.s095.t002","gold_sense_keys":["dream%1:09:01::"],"candidates":[{"index":1,"sense_key":"dream%1:09:03::","synset_id":"pipe_dream.n.01"},{"index":2,"sense_key":"dream%1:09:04::","synset_id":"dream.n.05"},{"index":3,"sense_key":"dream%1:26:00::","synset_id":"dream.n.06"},{"index":4,"sense_key":"dream%1:09:02::","synset_id":"dream.n.02"},{"index":5,"sense_key":"dream%1:12:00::","synset_id":"ambition.n.01"},{"index":6,"sense_key":"dream%1:09:01::","synset_id":"dream.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"dream%1:09:01::","call_ids":["senseval3.d000.s095.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"dream%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":124},"cost":{"currency":"USD","total_usd":0.006750000000000001,"input_uncached_usd":0.00255,"input_cached_usd":0.0,"output_usd":0.0042,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.589068374945782}
{"item_id":"senseval3.d000.s095.t003","gold_sense_keys":["dream%1:09:01::"],"candidates":[{"index":1,"sense_key":"dream%1:09:03::","synset_id":"pipe_dream.n.01"},{"index":2,"sense_key":"dream%1:26:00::","synset_id":"dream.n.06"},{"index":3,"sense_key":"dream%1:09:02::","synset_id":"dream.n.02"},{"index":4,"sense_key":"dream%1:09:04::","synset_id":"dream.n.05"},{"index":5,"sense_key":"dream%1:09:01::","synset_id":"dream.n.01"},{"index":6,"sense_key":"dream%1:12:00::","synset_id":"ambition.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"dream%1:09:01::","call_ids":["senseval3.d000.s095.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"dream%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":119},"cost":{"currency":"USD","total_usd":0.0066050000000000015,"input_uncached_usd":0.0025550000000000004,"input_cached_usd":0.0,"output_usd":0.00405,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.432641917024739}
{"item_id":"senseval3.d000.s095.t004","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":3,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":4,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":5,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"},{"index":6,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":9,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":10,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"begin%2:30:00::","call_ids":["senseval3.d000.s095.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"begin%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":777,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019695,"input_uncached_usd":0.0038850000000000004,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.900257125031203}
{"item_id":"senseval3.d000.s095.t005","gold_sense_keys":["overtone%1:10:00::"],"candidates":[{"index":1,"sense_key":"overtone%1:09:00::","synset_id":"overtone.n.02"},{"index":2,"sense_key":"overtone%1:10:00::","synset_id":"overtone.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"overtone%1:10:00::","call_ids":["senseval3.d000.s095.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"overtone%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":341,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.003235,"input_uncached_usd":0.0017050000000000001,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5718941669911146}
{"item_id":"senseval3.d000.s095.t006","gold_sense_keys":["nightmare%1:09:00::"],"candidates":[{"index":1,"sense_key":"nightmare%1:26:00::","synset_id":"nightmare.n.01"},{"index":2,"sense_key":"nightmare%1:09:00::","synset_id":"nightmare.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nightmare%1:26:00::","call_ids":["senseval3.d000.s095.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nightmare%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":320,"cached_input_tokens":0,"output_tokens":177,"reasoning_output_tokens":161},"cost":{"currency":"USD","total_usd":0.00691,"input_uncached_usd":0.0016,"input_cached_usd":0.0,"output_usd":0.0053100000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.799123791977763}
{"item_id":"senseval3.d000.s096.t000","gold_sense_keys":["know%2:31:01::","know%2:31:02::"],"candidates":[{"index":1,"sense_key":"know%2:31:14::","synset_id":"know.v.11"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":6,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:11::","synset_id":"know.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"know%2:31:02::","call_ids":["senseval3.d000.s096.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"know%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":779,"cached_input_tokens":0,"output_tokens":772,"reasoning_output_tokens":750},"cost":{"currency":"USD","total_usd":0.027055,"input_uncached_usd":0.0038950000000000005,"input_cached_usd":0.0,"output_usd":0.02316,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":16.49451416695956}
{"item_id":"senseval3.d000.s096.t001","gold_sense_keys":["dream%1:09:01::"],"candidates":[{"index":1,"sense_key":"dream%1:09:04::","synset_id":"dream.n.05"},{"index":2,"sense_key":"dream%1:12:00::","synset_id":"ambition.n.01"},{"index":3,"sense_key":"dream%1:26:00::","synset_id":"dream.n.06"},{"index":4,"sense_key":"dream%1:09:02::","synset_id":"dream.n.02"},{"index":5,"sense_key":"dream%1:09:03::","synset_id":"pipe_dream.n.01"},{"index":6,"sense_key":"dream%1:09:01::","synset_id":"dream.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"dream%1:09:01::","call_ids":["senseval3.d000.s096.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"dream%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":276,"reasoning_output_tokens":260},"cost":{"currency":"USD","total_usd":0.010805000000000002,"input_uncached_usd":0.0025250000000000003,"input_cached_usd":0.0,"output_usd":0.008280000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.762701249914244}
{"item_id":"senseval3.d000.s097.t000","gold_sense_keys":["tight%5:00:00:intoxicated:00"],"candidates":[{"index":1,"sense_key":"tight%5:00:00:fine:00","synset_id":"close.s.06"},{"index":2,"sense_key":"tight%5:00:00:tense:01","synset_id":"taut.s.01"},{"index":3,"sense_key":"tight%5:00:00:compact:00","synset_id":"tight.s.12"},{"index":4,"sense_key":"tight%5:00:00:equal:00","synset_id":"close.s.03"},{"index":5,"sense_key":"tight%5:00:00:intoxicated:00","synset_id":"besotted.s.01"},{"index":6,"sense_key":"tight%5:00:00:stingy:00","synset_id":"mean.s.06"},{"index":7,"sense_key":"tight%5:00:00:scarce:00","synset_id":"tight.s.05"},{"index":8,"sense_key":"tight%3:00:02::","synset_id":"tight.a.07"},{"index":9,"sense_key":"tight%5:00:00:difficult:00","synset_id":"nasty.s.01"},{"index":10,"sense_key":"tight%3:00:01::","synset_id":"tight.a.01"},{"index":11,"sense_key":"tight%5:00:00:invulnerable:00","synset_id":"tight.s.02"},{"index":12,"sense_key":"tight%5:00:00:closed:03","synset_id":"compressed.s.01"},{"index":13,"sense_key":"tight%5:00:00:demanding:01","synset_id":"rigorous.s.02"},{"index":14,"sense_key":"tight%5:00:00:secure:03","synset_id":"tight.s.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"tight%5:00:00:intoxicated:00","call_ids":["senseval3.d000.s097.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"tight%5:00:00:intoxicated:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.006240000000000001,"input_uncached_usd":0.004260000000000001,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9835088329855353}
{"item_id":"senseval3.d000.s097.t001","gold_sense_keys":["know%2:31:01::","know%2:31:02::"],"candidates":[{"index":1,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":2,"sense_key":"know%2:31:14::","synset_id":"know.v.11"},{"index":3,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":4,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":5,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":6,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":7,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":8,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":9,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":10,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":11,"sense_key":"know%2:31:11::","synset_id":"know.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval3.d000.s097.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":784,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":112},"cost":{"currency":"USD","total_usd":0.007760000000000001,"input_uncached_usd":0.003920000000000001,"input_cached_usd":0.0,"output_usd":0.00384,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.592421790934168}
{"item_id":"senseval3.d000.s098.t000","gold_sense_keys":["same%3:00:02::"],"candidates":[{"index":1,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"},{"index":2,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":3,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":4,"sense_key":"same%3:00:02::","synset_id":"same.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"same%3:00:02::","call_ids":["senseval3.d000.s098.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"same%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.00448,"input_uncached_usd":0.0022,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7035309160128236}
{"item_id":"senseval3.d000.s098.t001","gold_sense_keys":["bar%1:06:04::"],"candidates":[{"index":1,"sense_key":"bar%1:04:00::","synset_id":"prevention.n.01"},{"index":2,"sense_key":"bar%1:06:07::","synset_id":"bar.n.15"},{"index":3,"sense_key":"bar%1:07:00::","synset_id":"stripe.n.05"},{"index":4,"sense_key":"bar%1:06:01::","synset_id":"browning_automatic_rifle.n.01"},{"index":5,"sense_key":"bar%1:14:00::","synset_id":"legal_profession.n.01"},{"index":6,"sense_key":"bar%1:06:02::","synset_id":"bar.n.05"},{"index":7,"sense_key":"bar%1:23:00::","synset_id":"bar.n.07"},{"index":8,"sense_key":"bar%1:06:08::","synset_id":"bar.n.14"},{"index":9,"sense_key":"bar%1:06:06::","synset_id":"cake.n.01"},{"index":10,"sense_key":"bar%1:06:09::","synset_id":"bar.n.13"},{"index":11,"sense_key":"bar%1:10:00::","synset_id":"measure.n.07"},{"index":12,"sense_key":"bar%1:06:05::","synset_id":"bar.n.02"},{"index":13,"sense_key":"bar%1:17:00::","synset_id":"bar.n.08"},{"index":14,"sense_key":"bar%1:06:00::","synset_id":"bar.n.03"},{"index":15,"sense_key":"bar%1:06:04::","synset_id":"barroom.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"bar%1:06:04::","call_ids":["senseval3.d000.s098.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"bar%1:06:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":964,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":107},"cost":{"currency":"USD","total_usd":0.008480000000000001,"input_uncached_usd":0.0048200000000000005,"input_cached_usd":0.0,"output_usd":0.00366,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.998792125028558}
{"item_id":"senseval3.d000.s098.t002","gold_sense_keys":["week%1:28:00::"],"candidates":[{"index":1,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":2,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"week%1:28:00::","call_ids":["senseval3.d000.s098.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"week%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.004515,"input_uncached_usd":0.001935,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3597439160803333}
{"item_id":"senseval3.d000.s098.t003","gold_sense_keys":["night%1:28:03::"],"candidates":[{"index":1,"sense_key":"night%1:28:00::","synset_id":"night.n.01"},{"index":2,"sense_key":"night%1:28:03::","synset_id":"night.n.07"},{"index":3,"sense_key":"night%1:28:04::","synset_id":"night.n.03"},{"index":4,"sense_key":"night%1:28:01::","synset_id":"night.n.02"},{"index":5,"sense_key":"night%1:18:00::","synset_id":"nox.n.01"},{"index":6,"sense_key":"night%1:28:02::","synset_id":"night.n.04"},{"index":7,"sense_key":"night%1:26:00::","synset_id":"night.n.05"},{"index":8,"sense_key":"night%1:28:05::","synset_id":"night.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"night%1:28:03::","call_ids":["senseval3.d000.s098.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"night%1:28:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":7903,"reasoning_output_tokens":7796},"cost":{"currency":"USD","total_usd":0.23973,"input_uncached_usd":0.0026400000000000004,"input_cached_usd":0.0,"output_usd":0.23709,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":116.50193962501362}
{"item_id":"senseval3.d000.s098.t004","gold_sense_keys":["have%2:40:00::"],"candidates":[{"index":1,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":2,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":3,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":4,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":5,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":6,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":7,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":8,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":9,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":10,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":11,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":12,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":13,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":14,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":15,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":16,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":17,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":18,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d000.s098.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1114,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":140},"cost":{"currency":"USD","total_usd":0.01025,"input_uncached_usd":0.00557,"input_cached_usd":0.0,"output_usd":0.00468,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5728000410599634}
{"item_id":"senseval3.d000.s098.t005","gold_sense_keys":["excuse%1:10:00::"],"candidates":[{"index":1,"sense_key":"excuse%1:10:01::","synset_id":"excuse.n.02"},{"index":2,"sense_key":"excuse%1:09:00::","synset_id":"apology.n.03"},{"index":3,"sense_key":"excuse%1:10:00::","synset_id":"excuse.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"excuse%1:10:00::","call_ids":["senseval3.d000.s098.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"excuse%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.00421,"input_uncached_usd":0.00214,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6184995420044288}
{"item_id":"senseval3.d000.s098.t006","gold_sense_keys":["drink%2:34:01::"],"candidates":[{"index":1,"sense_key":"drink%2:34:02::","synset_id":"drink.v.05"},{"index":2,"sense_key":"drink%2:34:00::","synset_id":"drink.v.01"},{"index":3,"sense_key":"drink%2:34:12::","synset_id":"toast.v.02"},{"index":4,"sense_key":"drink%2:31:00::","synset_id":"drink_in.v.01"},{"index":5,"sense_key":"drink%2:34:01::","synset_id":"drink.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"drink%2:34:01::","call_ids":["senseval3.d000.s098.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"drink%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":119},"cost":{"currency":"USD","total_usd":0.006530000000000001,"input_uncached_usd":0.00248,"input_cached_usd":0.0,"output_usd":0.00405,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.140362916048616}
{"item_id":"senseval3.d000.s098.t007","gold_sense_keys":["usual%3:00:00::"],"candidates":[{"index":1,"sense_key":"usual%5:00:00:familiar:02","synset_id":"common.s.02"},{"index":2,"sense_key":"usual%3:00:00::","synset_id":"usual.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"usual%3:00:00::","call_ids":["senseval3.d000.s098.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"usual%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":364,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.0044,"input_uncached_usd":0.0018200000000000002,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5971631249412894}
{"item_id":"senseval3.d000.s099.t000","gold_sense_keys":["sit%2:35:00::","sit%2:38:00::"],"candidates":[{"index":1,"sense_key":"sit%2:38:03::","synset_id":"ride.v.01"},{"index":2,"sense_key":"sit%2:38:00::","synset_id":"sit_down.v.01"},{"index":3,"sense_key":"sit%2:35:01::","synset_id":"seat.v.01"},{"index":4,"sense_key":"sit%2:41:05::","synset_id":"baby-sit.v.02"},{"index":5,"sense_key":"sit%2:42:00::","synset_id":"sit.v.02"},{"index":6,"sense_key":"sit%2:38:01::","synset_id":"sit.v.04"},{"index":7,"sense_key":"sit%2:33:00::","synset_id":"sit.v.10"},{"index":8,"sense_key":"sit%2:39:00::","synset_id":"model.v.03"},{"index":9,"sense_key":"sit%2:35:00::","synset_id":"sit.v.01"},{"index":10,"sense_key":"sit%2:42:05::","synset_id":"sit.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"sit%2:35:00::","call_ids":["senseval3.d000.s099.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"sit%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":676,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.00686,"input_uncached_usd":0.00338,"input_cached_usd":0.0,"output_usd":0.00348,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5381631669588387}
{"item_id":"senseval3.d000.s099.t001","gold_sense_keys":["usual%3:00:00::"],"candidates":[{"index":1,"sense_key":"usual%3:00:00::","synset_id":"usual.a.01"},{"index":2,"sense_key":"usual%5:00:00:familiar:02","synset_id":"common.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"usual%3:00:00::","call_ids":["senseval3.d000.s099.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"usual%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":376,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.00398,"input_uncached_usd":0.0018800000000000002,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9410191250499338}
{"item_id":"senseval3.d000.s099.t002","gold_sense_keys":["corner%1:06:02::"],"candidates":[{"index":1,"sense_key":"corner%1:15:02::","synset_id":"corner.n.01"},{"index":2,"sense_key":"corner%1:15:03::","synset_id":"corner.n.10"},{"index":3,"sense_key":"corner%1:06:00::","synset_id":"corner.n.04"},{"index":4,"sense_key":"corner%1:17:00::","synset_id":"corner.n.09"},{"index":5,"sense_key":"corner%1:26:01::","synset_id":"corner.n.08"},{"index":6,"sense_key":"corner%1:15:00::","synset_id":"corner.n.02"},{"index":7,"sense_key":"corner%1:06:01::","synset_id":"corner.n.11"},{"index":8,"sense_key":"corner%1:26:00::","synset_id":"corner.n.07"},{"index":9,"sense_key":"corner%1:25:00::","synset_id":"recess.n.02"},{"index":10,"sense_key":"corner%1:15:01::","synset_id":"corner.n.05"},{"index":11,"sense_key":"corner%1:06:02::","synset_id":"corner.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"corner%1:06:02::","call_ids":["senseval3.d000.s099.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"corner%1:06:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.0059900000000000005,"input_uncached_usd":0.0034400000000000003,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6304327920079231}
{"item_id":"senseval3.d000.s099.t003","gold_sense_keys":["little%3:00:01::"],"candidates":[{"index":1,"sense_key":"little%3:00:01::","synset_id":"small.a.01"},{"index":2,"sense_key":"little%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":3,"sense_key":"little%5:00:00:soft:04","synset_id":"little.s.03"},{"index":4,"sense_key":"little%5:00:00:unimportant:00","synset_id":"fiddling.s.01"},{"index":5,"sense_key":"little%3:00:00::","synset_id":"short.a.03"},{"index":6,"sense_key":"little%3:00:03::","synset_id":"little.a.02"},{"index":7,"sense_key":"little%5:00:00:young:00","synset_id":"little.s.01"},{"index":8,"sense_key":"little%5:00:00:emotional:00","synset_id":"little.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"little%3:00:01::","call_ids":["senseval3.d000.s099.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"little%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":674,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.0061600000000000005,"input_uncached_usd":0.00337,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9486154170008376}
{"item_id":"senseval3.d000.s099.t004","gold_sense_keys":["table%1:06:01::"],"candidates":[{"index":1,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":2,"sense_key":"table%1:06:02::","synset_id":"table.n.03"},{"index":3,"sense_key":"table%1:14:00::","synset_id":"table.n.01"},{"index":4,"sense_key":"table%1:14:01::","synset_id":"table.n.05"},{"index":5,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":6,"sense_key":"table%1:13:01::","synset_id":"board.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"table%1:06:01::","call_ids":["senseval3.d000.s099.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"table%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.005835,"input_uncached_usd":0.0026850000000000003,"input_cached_usd":0.0,"output_usd":0.00315,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6802832920802757}
{"item_id":"senseval3.d000.s099.t006","gold_sense_keys":["talkative%5:00:00:communicative:00","talkative%5:00:00:voluble:00"],"candidates":[{"index":1,"sense_key":"talkative%5:00:00:voluble:00","synset_id":"chatty.s.01"},{"index":2,"sense_key":"talkative%5:00:00:indiscreet:00","synset_id":"bigmouthed.s.01"},{"index":3,"sense_key":"talkative%5:00:00:communicative:00","synset_id":"expansive.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"talkative%5:00:00:voluble:00","call_ids":["senseval3.d000.s099.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"talkative%5:00:00:voluble:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":438,"cached_input_tokens":0,"output_tokens":238,"reasoning_output_tokens":222},"cost":{"currency":"USD","total_usd":0.009330000000000001,"input_uncached_usd":0.00219,"input_cached_usd":0.0,"output_usd":0.0071400000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.0716282919747755}
{"item_id":"senseval3.d000.s099.t007","gold_sense_keys":["friendly%3:00:01::"],"candidates":[{"index":1,"sense_key":"friendly%3:00:01::","synset_id":"friendly.a.01"},{"index":2,"sense_key":"friendly%3:00:03::","synset_id":"friendly.a.03"},{"index":3,"sense_key":"friendly%3:00:02::","synset_id":"friendly.a.04"},{"index":4,"sense_key":"friendly%5:00:00:amicable:00","synset_id":"friendly.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"friendly%3:00:01::","call_ids":["senseval3.d000.s099.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"friendly%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":263,"reasoning_output_tokens":247},"cost":{"currency":"USD","total_usd":0.01015,"input_uncached_usd":0.0022600000000000003,"input_cached_usd":0.0,"output_usd":0.00789,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.2365282089449465}
{"item_id":"senseval3.d000.s100.t000","gold_sense_keys":["enjoy%2:34:00::","enjoy%2:37:00::","enjoy%2:37:01::"],"candidates":[{"index":1,"sense_key":"enjoy%2:34:01::","synset_id":"enjoy.v.02"},{"index":2,"sense_key":"enjoy%2:39:00::","synset_id":"enjoy.v.04"},{"index":3,"sense_key":"enjoy%2:37:00::","synset_id":"enjoy.v.01"},{"index":4,"sense_key":"enjoy%2:37:01::","synset_id":"love.v.02"},{"index":5,"sense_key":"enjoy%2:34:00::","synset_id":"delight.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"enjoy%2:37:00::","call_ids":["senseval3.d000.s100.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"enjoy%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":168,"reasoning_output_tokens":152},"cost":{"currency":"USD","total_usd":0.007300000000000001,"input_uncached_usd":0.0022600000000000003,"input_cached_usd":0.0,"output_usd":0.00504,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3695752499625087}
{"item_id":"senseval3.d000.s100.t002","gold_sense_keys":["ritual%1:04:00::"],"candidates":[{"index":1,"sense_key":"ritual%1:04:02::","synset_id":"ritual.n.03"},{"index":2,"sense_key":"ritual%1:04:01::","synset_id":"ritual.n.02"},{"index":3,"sense_key":"ritual%1:04:00::","synset_id":"ritual.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ritual%1:04:00::","call_ids":["senseval3.d000.s100.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ritual%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":355,"cached_input_tokens":0,"output_tokens":217,"reasoning_output_tokens":202},"cost":{"currency":"USD","total_usd":0.008285,"input_uncached_usd":0.001775,"input_cached_usd":0.0,"output_usd":0.00651,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.232872291002423}
{"item_id":"senseval3.d000.s100.t003","gold_sense_keys":["wash_down%2:34:00::"],"candidates":[{"index":1,"sense_key":"wash_down%2:34:00::","synset_id":"wash_down.v.01"},{"index":2,"sense_key":"wash_down%2:30:00::","synset_id":"wash_down.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wash_down%2:34:00::","call_ids":["senseval3.d000.s100.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wash_down%2:34:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.00338,"input_uncached_usd":0.00185,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5269816670333967}
{"item_id":"senseval3.d000.s100.t004","gold_sense_keys":["pet%1:18:00::"],"candidates":[{"index":1,"sense_key":"pet%1:05:00::","synset_id":"pet.n.01"},{"index":2,"sense_key":"pet%1:18:00::","synset_id":"darling.n.01"},{"index":3,"sense_key":"pet%1:04:00::","synset_id":"positron_emission_tomography.n.01"},{"index":4,"sense_key":"pet%1:12:00::","synset_id":"pet.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pet%1:18:00::","call_ids":["senseval3.d000.s100.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pet%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":421,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":133},"cost":{"currency":"USD","total_usd":0.006575000000000001,"input_uncached_usd":0.002105,"input_cached_usd":0.0,"output_usd":0.00447,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.598258082987741}
{"item_id":"senseval3.d000.s100.t005","gold_sense_keys":["grievance%1:12:00::"],"candidates":[{"index":1,"sense_key":"grievance%1:10:00::","synset_id":"grievance.n.03"},{"index":2,"sense_key":"grievance%1:12:00::","synset_id":"grudge.n.01"},{"index":3,"sense_key":"grievance%1:10:01::","synset_id":"grievance.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"grievance%1:10:00::","call_ids":["senseval3.d000.s100.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"grievance%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":395,"cached_input_tokens":0,"output_tokens":305,"reasoning_output_tokens":290},"cost":{"currency":"USD","total_usd":0.011125,"input_uncached_usd":0.001975,"input_cached_usd":0.0,"output_usd":0.00915,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.9981578750303015}
{"item_id":"senseval3.d000.s100.t006","gold_sense_keys":["bourbon%1:13:00::"],"candidates":[{"index":1,"sense_key":"bourbon%1:14:00::","synset_id":"bourbon.n.04"},{"index":2,"sense_key":"bourbon%1:18:01::","synset_id":"bourbon.n.01"},{"index":3,"sense_key":"bourbon%1:13:00::","synset_id":"bourbon.n.02"},{"index":4,"sense_key":"bourbon%1:18:00::","synset_id":"bourbon.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bourbon%1:13:00::","call_ids":["senseval3.d000.s100.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bourbon%1:13:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.0034750000000000002,"input_uncached_usd":0.002095,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9232888750266284}
{"item_id":"senseval3.d000.s100.t007","gold_sense_keys":["water%1:13:00::"],"candidates":[{"index":1,"sense_key":"water%1:06:00::","synset_id":"water_system.n.02"},{"index":2,"sense_key":"water%1:27:00::","synset_id":"water.n.01"},{"index":3,"sense_key":"water%1:27:01::","synset_id":"urine.n.01"},{"index":4,"sense_key":"water%1:17:00::","synset_id":"body_of_water.n.01"},{"index":5,"sense_key":"water%1:13:00::","synset_id":"water.n.06"},{"index":6,"sense_key":"water%1:27:02::","synset_id":"water.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"water%1:13:00::","call_ids":["senseval3.d000.s100.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"water%1:13:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":580,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01874,"input_uncached_usd":0.0029000000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.498688000021502}
{"item_id":"senseval3.d000.s101.t000","gold_sense_keys":["favorite%5:00:00:loved:00"],"candidates":[{"index":1,"sense_key":"favorite%5:00:00:popular:00","synset_id":"favorite.s.01"},{"index":2,"sense_key":"favorite%5:00:00:loved:00","synset_id":"favored.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"favorite%5:00:00:loved:00","call_ids":["senseval3.d000.s101.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"favorite%5:00:00:loved:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":349,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.0040550000000000004,"input_uncached_usd":0.0017450000000000002,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9761726250872016}
{"item_id":"senseval3.d000.s101.t001","gold_sense_keys":["grievance%1:12:00::"],"candidates":[{"index":1,"sense_key":"grievance%1:10:01::","synset_id":"grievance.n.02"},{"index":2,"sense_key":"grievance%1:12:00::","synset_id":"grudge.n.01"},{"index":3,"sense_key":"grievance%1:10:00::","synset_id":"grievance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"grievance%1:10:00::","call_ids":["senseval3.d000.s101.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"grievance%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":379,"cached_input_tokens":0,"output_tokens":302,"reasoning_output_tokens":286},"cost":{"currency":"USD","total_usd":0.010955000000000001,"input_uncached_usd":0.0018950000000000002,"input_cached_usd":0.0,"output_usd":0.00906,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.46135245799087}
{"item_id":"senseval3.d000.s102.t000","gold_sense_keys":["forget%2:31:01::"],"candidates":[{"index":1,"sense_key":"forget%2:31:01::","synset_id":"forget.v.01"},{"index":2,"sense_key":"forget%2:31:00::","synset_id":"forget.v.02"},{"index":3,"sense_key":"forget%2:31:02::","synset_id":"forget.v.04"},{"index":4,"sense_key":"forget%2:31:03::","synset_id":"forget.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"forget%2:31:01::","call_ids":["senseval3.d000.s102.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"forget%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":113},"cost":{"currency":"USD","total_usd":0.006025000000000001,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.00384,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5218729169573635}
{"item_id":"senseval3.d000.s103.t000","gold_sense_keys":["mention%2:32:02::"],"candidates":[{"index":1,"sense_key":"mention%2:32:01::","synset_id":"mention.v.03"},{"index":2,"sense_key":"mention%2:32:00::","synset_id":"note.v.01"},{"index":3,"sense_key":"mention%2:32:02::","synset_id":"mention.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"mention%2:32:02::","call_ids":["senseval3.d000.s103.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"mention%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":130},"cost":{"currency":"USD","total_usd":0.006300000000000001,"input_uncached_usd":0.0019500000000000001,"input_cached_usd":0.0,"output_usd":0.00435,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.508943916996941}
{"item_id":"senseval3.d000.s104.t000","gold_sense_keys":["real%3:00:02::","real%5:00:00:true:00"],"candidates":[{"index":1,"sense_key":"real%5:00:00:true:00","synset_id":"actual.s.02"},{"index":2,"sense_key":"real%3:00:02::","synset_id":"real.a.02"},{"index":3,"sense_key":"real%5:00:00:concrete:00","synset_id":"real.s.02"},{"index":4,"sense_key":"real%3:00:03::","synset_id":"substantial.a.03"},{"index":5,"sense_key":"real%5:00:00:serious:00","synset_id":"real.s.01"},{"index":6,"sense_key":"real%3:00:00::","synset_id":"real.a.01"},{"index":7,"sense_key":"real%5:00:00:tangible:02","synset_id":"real.s.04"},{"index":8,"sense_key":"real%5:00:02:realistic:00","synset_id":"veridical.s.01"},{"index":9,"sense_key":"real%3:00:01::","synset_id":"real.a.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"real%5:00:00:true:00","call_ids":["senseval3.d000.s104.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"real%5:00:00:true:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":713,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019405,"input_uncached_usd":0.0035650000000000005,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.007587166968733}
{"item_id":"senseval3.d000.s104.t001","gold_sense_keys":["grievance%1:12:00::"],"candidates":[{"index":1,"sense_key":"grievance%1:10:00::","synset_id":"grievance.n.03"},{"index":2,"sense_key":"grievance%1:12:00::","synset_id":"grudge.n.01"},{"index":3,"sense_key":"grievance%1:10:01::","synset_id":"grievance.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"grievance%1:12:00::","call_ids":["senseval3.d000.s104.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"grievance%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017785,"input_uncached_usd":0.0019450000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.35870637500193}
{"item_id":"senseval3.d000.s105.t000","gold_sense_keys":["end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:04:02::","synset_id":"end.n.14"},{"index":2,"sense_key":"end%1:18:00::","synset_id":"end.n.08"},{"index":3,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":4,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":5,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":6,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":7,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"},{"index":8,"sense_key":"end%1:15:02::","synset_id":"end.n.07"},{"index":9,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":10,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":11,"sense_key":"end%1:11:00::","synset_id":"end.n.03"},{"index":12,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":13,"sense_key":"end%1:15:01::","synset_id":"end.n.09"},{"index":14,"sense_key":"end%1:15:00::","synset_id":"end.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"end%1:11:00::","call_ids":["senseval3.d000.s105.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"end%1:11:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":872,"cached_input_tokens":0,"output_tokens":1957,"reasoning_output_tokens":1923},"cost":{"currency":"USD","total_usd":0.06307,"input_uncached_usd":0.00436,"input_cached_usd":0.0,"output_usd":0.05871,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":29.39754570904188}
{"item_id":"senseval3.d000.s105.t003","gold_sense_keys":["make%2:36:00::","make%2:36:08::"],"candidates":[{"index":1,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":2,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":3,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":4,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":5,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":6,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":7,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":8,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":11,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":12,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":13,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":14,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":15,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":16,"sense_key":"make%2:29:01::","synset_id":"make.v.49"},{"index":17,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":18,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":19,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":20,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":21,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":22,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":23,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":24,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":25,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":26,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":27,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":28,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":29,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":30,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":31,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":32,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":33,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":34,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":35,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":36,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":37,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":38,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":39,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":40,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":41,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":42,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":43,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":44,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":45,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":46,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":47,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":48,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":49,"sense_key":"make%2:39:12::","synset_id":"make.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":20,"chosen_sense_key":"make%2:36:00::","call_ids":["senseval3.d000.s105.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":20,"predicted_sense_key":"make%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2235,"cached_input_tokens":0,"output_tokens":360,"reasoning_output_tokens":345},"cost":{"currency":"USD","total_usd":0.021975,"input_uncached_usd":0.011175000000000001,"input_cached_usd":0.0,"output_usd":0.0108,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.034074957948178}
{"item_id":"senseval3.d000.s105.t004","gold_sense_keys":["wet%3:00:01::"],"candidates":[{"index":1,"sense_key":"wet%3:00:01::","synset_id":"wet.a.01"},{"index":2,"sense_key":"wet%5:00:00:intoxicated:00","synset_id":"besotted.s.01"},{"index":3,"sense_key":"wet%3:00:05::","synset_id":"wet.a.02"},{"index":4,"sense_key":"wet%5:00:00:alcoholic:00","synset_id":"wet.s.01"},{"index":5,"sense_key":"wet%3:00:02::","synset_id":"wet.a.04"},{"index":6,"sense_key":"wet%3:00:04::","synset_id":"wet.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wet%3:00:01::","call_ids":["senseval3.d000.s105.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wet%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":80},"cost":{"currency":"USD","total_usd":0.00541,"input_uncached_usd":0.00253,"input_cached_usd":0.0,"output_usd":0.00288,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7509347080485895}
{"item_id":"senseval3.d000.s105.t005","gold_sense_keys":["ring%1:25:00::"],"candidates":[{"index":1,"sense_key":"ring%1:06:02::","synset_id":"hoop.n.02"},{"index":2,"sense_key":"ring%1:06:01::","synset_id":"band.n.12"},{"index":3,"sense_key":"ring%1:25:00::","synset_id":"ring.n.02"},{"index":4,"sense_key":"ring%1:17:00::","synset_id":"closed_chain.n.01"},{"index":5,"sense_key":"ring%1:11:00::","synset_id":"ring.n.06"},{"index":6,"sense_key":"ring%1:06:03::","synset_id":"ring.n.07"},{"index":7,"sense_key":"ring%1:06:00::","synset_id":"ring.n.08"},{"index":8,"sense_key":"ring%1:14:00::","synset_id":"gang.n.01"},{"index":9,"sense_key":"ring%1:07:00::","synset_id":"ring.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ring%1:25:00::","call_ids":["senseval3.d000.s105.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ring%1:25:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":679,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.005855000000000001,"input_uncached_usd":0.0033950000000000004,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.238597916904837}
{"item_id":"senseval3.d000.s105.t006","gold_sense_keys":["bottom%1:15:00::"],"candidates":[{"index":1,"sense_key":"bottom%1:28:00::","synset_id":"bottom.n.04"},{"index":2,"sense_key":"bottom%1:17:01::","synset_id":"bottomland.n.01"},{"index":3,"sense_key":"bottom%1:15:01::","synset_id":"bottom.n.02"},{"index":4,"sense_key":"bottom%1:06:00::","synset_id":"bottom.n.07"},{"index":5,"sense_key":"bottom%1:17:00::","synset_id":"bed.n.03"},{"index":6,"sense_key":"bottom%1:15:00::","synset_id":"bottom.n.01"},{"index":7,"sense_key":"bottom%1:08:00::","synset_id":"buttocks.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"bottom%1:15:00::","call_ids":["senseval3.d000.s105.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"bottom%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":115},"cost":{"currency":"USD","total_usd":0.006755000000000001,"input_uncached_usd":0.0028250000000000003,"input_cached_usd":0.0,"output_usd":0.00393,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9080916659440845}
{"item_id":"senseval3.d000.s105.t007","gold_sense_keys":["glass%1:06:00::"],"candidates":[{"index":1,"sense_key":"glass%1:06:02::","synset_id":"looking_glass.n.01"},{"index":2,"sense_key":"glass%1:27:00::","synset_id":"glass.n.01"},{"index":3,"sense_key":"glass%1:06:03::","synset_id":"field_glass.n.01"},{"index":4,"sense_key":"glass%1:06:01::","synset_id":"methamphetamine.n.01"},{"index":5,"sense_key":"glass%1:23:00::","synset_id":"glass.n.03"},{"index":6,"sense_key":"glass%1:06:00::","synset_id":"glass.n.02"},{"index":7,"sense_key":"glass%1:06:04::","synset_id":"glass.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"glass%1:06:00::","call_ids":["senseval3.d000.s105.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"glass%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.004305,"input_uncached_usd":0.002595,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.07790966599714}
{"item_id":"senseval3.d000.s105.t008","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:42:03::","synset_id":"become.v.04"},{"index":2,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":3,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":4,"sense_key":"become%2:30:00::","synset_id":"become.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"become%2:30:00::","call_ids":["senseval3.d000.s105.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"become%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.005155,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.023653083946556}
{"item_id":"senseval3.d000.s105.t009","gold_sense_keys":["aware%3:00:00::"],"candidates":[{"index":1,"sense_key":"aware%3:00:00::","synset_id":"aware.a.01"},{"index":2,"sense_key":"aware%3:00:04::","synset_id":"mindful.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"aware%3:00:00::","call_ids":["senseval3.d000.s105.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"aware%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":377,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.0035050000000000003,"input_uncached_usd":0.0018850000000000002,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2143416670151055}
{"item_id":"senseval3.d000.s106.t000","gold_sense_keys":["look%2:39:00::"],"candidates":[{"index":1,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":2,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":3,"sense_key":"look%2:39:01::","synset_id":"look.v.02"},{"index":4,"sense_key":"look%2:31:02::","synset_id":"count.v.08"},{"index":5,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":6,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":7,"sense_key":"look%2:39:02::","synset_id":"search.v.02"},{"index":8,"sense_key":"look%2:29:00::","synset_id":"look.v.03"},{"index":9,"sense_key":"look%2:42:00::","synset_id":"front.v.01"},{"index":10,"sense_key":"look%2:39:00::","synset_id":"look.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"look%2:39:00::","call_ids":["senseval3.d000.s106.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"look%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.0057350000000000005,"input_uncached_usd":0.003605,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5961915409425274}
{"item_id":"senseval3.d000.s106.t002","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":2,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":3,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":4,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s106.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.00387,"input_uncached_usd":0.00237,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3898517090128735}
{"item_id":"senseval3.d000.s106.t004","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"},{"index":2,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s106.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":340,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":85},"cost":{"currency":"USD","total_usd":0.00473,"input_uncached_usd":0.0017000000000000001,"input_cached_usd":0.0,"output_usd":0.00303,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0988852500449866}
{"item_id":"senseval3.d000.s106.t005","gold_sense_keys":["sit%2:35:00::"],"candidates":[{"index":1,"sense_key":"sit%2:39:00::","synset_id":"model.v.03"},{"index":2,"sense_key":"sit%2:38:03::","synset_id":"ride.v.01"},{"index":3,"sense_key":"sit%2:38:00::","synset_id":"sit_down.v.01"},{"index":4,"sense_key":"sit%2:42:00::","synset_id":"sit.v.02"},{"index":5,"sense_key":"sit%2:41:05::","synset_id":"baby-sit.v.02"},{"index":6,"sense_key":"sit%2:42:05::","synset_id":"sit.v.07"},{"index":7,"sense_key":"sit%2:35:01::","synset_id":"seat.v.01"},{"index":8,"sense_key":"sit%2:33:00::","synset_id":"sit.v.10"},{"index":9,"sense_key":"sit%2:35:00::","synset_id":"sit.v.01"},{"index":10,"sense_key":"sit%2:38:01::","synset_id":"sit.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"sit%2:35:00::","call_ids":["senseval3.d000.s106.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"sit%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":664,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":138},"cost":{"currency":"USD","total_usd":0.007940000000000001,"input_uncached_usd":0.0033200000000000005,"input_cached_usd":0.0,"output_usd":0.00462,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.694244584068656}
{"item_id":"senseval3.d000.s106.t006","gold_sense_keys":["table%1:06:01::"],"candidates":[{"index":1,"sense_key":"table%1:14:00::","synset_id":"table.n.01"},{"index":2,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":3,"sense_key":"table%1:14:01::","synset_id":"table.n.05"},{"index":4,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":5,"sense_key":"table%1:06:02::","synset_id":"table.n.03"},{"index":6,"sense_key":"table%1:13:01::","synset_id":"board.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"table%1:06:01::","call_ids":["senseval3.d000.s106.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"table%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.004215,"input_uncached_usd":0.002625,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4626562499906868}
{"item_id":"senseval3.d000.s106.t008","gold_sense_keys":["secret%5:00:03:covert:00"],"candidates":[{"index":1,"sense_key":"secret%5:00:00:private:00","synset_id":"confidential.s.02"},{"index":2,"sense_key":"secret%5:00:01:covert:00","synset_id":"clandestine.s.01"},{"index":3,"sense_key":"secret%5:00:00:esoteric:00","synset_id":"mysterious.s.02"},{"index":4,"sense_key":"secret%5:00:02:private:00","synset_id":"privy.s.01"},{"index":5,"sense_key":"secret%5:00:02:concealed:00","synset_id":"hidden.s.02"},{"index":6,"sense_key":"secret%5:00:00:unacknowledged:00","synset_id":"unavowed.s.03"},{"index":7,"sense_key":"secret%5:00:03:covert:00","synset_id":"secret.s.04"},{"index":8,"sense_key":"secret%5:00:00:concealed:00","synset_id":"secret.s.01"},{"index":9,"sense_key":"secret%5:00:02:covert:00","synset_id":"secret.s.09"},{"index":10,"sense_key":"secret%5:00:00:inward:00","synset_id":"secret.s.05"},{"index":11,"sense_key":"secret%5:00:00:classified:02","synset_id":"secret.s.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"secret%5:00:00:esoteric:00","call_ids":["senseval3.d000.s106.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"secret%5:00:00:esoteric:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":782,"cached_input_tokens":0,"output_tokens":2081,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.06634,"input_uncached_usd":0.00391,"input_cached_usd":0.0,"output_usd":0.06243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":42.94787625002209}
{"item_id":"senseval3.d000.s107.t000","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"},{"index":2,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d000.s107.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01828,"input_uncached_usd":0.0024400000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.757965791970491}
{"item_id":"senseval3.d000.s107.t001","gold_sense_keys":["name%1:10:00::"],"candidates":[{"index":1,"sense_key":"name%1:10:00::","synset_id":"name.n.01"},{"index":2,"sense_key":"name%1:14:00::","synset_id":"name.n.03"},{"index":3,"sense_key":"name%1:26:00::","synset_id":"name.n.02"},{"index":4,"sense_key":"name%1:18:00::","synset_id":"name.n.04"},{"index":5,"sense_key":"name%1:04:00::","synset_id":"name.n.05"},{"index":6,"sense_key":"name%1:10:01::","synset_id":"name.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"name%1:10:00::","call_ids":["senseval3.d000.s107.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"name%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.004235000000000001,"input_uncached_usd":0.0025550000000000004,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7893255000235513}
{"item_id":"senseval3.d000.s107.t002","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":4,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":5,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":6,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":7,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":8,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"},{"index":9,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":16,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":17,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":18,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":19,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":20,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":21,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":22,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":23,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":24,"sense_key":"see%2:31:00::","synset_id":"see.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval3.d000.s107.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1431,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.022965000000000003,"input_uncached_usd":0.007155000000000001,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.344331457978114}
{"item_id":"senseval3.d000.s107.t003","gold_sense_keys":["sit_down%2:38:00::"],"candidates":[{"index":1,"sense_key":"sit_down%2:35:03::","synset_id":"sit.v.01"},{"index":2,"sense_key":"sit_down%2:35:00::","synset_id":"seat.v.01"},{"index":3,"sense_key":"sit_down%2:38:00::","synset_id":"sit_down.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"sit_down%2:38:00::","call_ids":["senseval3.d000.s107.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"sit_down%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":382,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.004070000000000001,"input_uncached_usd":0.0019100000000000002,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.065343874972314}
{"item_id":"senseval3.d000.s108.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":2,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":3,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":4,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":5,"sense_key":"man%1:05:00::","synset_id":"world.n.08"},{"index":6,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":7,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":8,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":9,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":10,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":11,"sense_key":"man%1:15:00::","synset_id":"man.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s108.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":842,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.00625,"input_uncached_usd":0.00421,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.26511566597037}
{"item_id":"senseval3.d000.s108.t001","gold_sense_keys":["uncanny%5:00:00:supernatural:00"],"candidates":[{"index":1,"sense_key":"uncanny%5:00:00:supernatural:00","synset_id":"eldritch.s.01"},{"index":2,"sense_key":"uncanny%5:00:00:extraordinary:00","synset_id":"preternatural.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"uncanny%5:00:00:supernatural:00","call_ids":["senseval3.d000.s108.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"uncanny%5:00:00:supernatural:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":436,"cached_input_tokens":0,"output_tokens":165,"reasoning_output_tokens":150},"cost":{"currency":"USD","total_usd":0.007130000000000001,"input_uncached_usd":0.00218,"input_cached_usd":0.0,"output_usd":0.00495,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.433597082970664}
{"item_id":"senseval3.d000.s108.t002","gold_sense_keys":["shadow%1:09:00::","shadow%1:26:01::"],"candidates":[{"index":1,"sense_key":"shadow%1:18:01::","synset_id":"shadow.n.09"},{"index":2,"sense_key":"shadow%1:12:00::","synset_id":"shadow.n.04"},{"index":3,"sense_key":"shadow%1:26:00::","synset_id":"shadow.n.07"},{"index":4,"sense_key":"shadow%1:07:00::","synset_id":"shadow.n.06"},{"index":5,"sense_key":"shadow%1:18:00::","synset_id":"tail.n.05"},{"index":6,"sense_key":"shadow%1:10:00::","synset_id":"trace.n.02"},{"index":7,"sense_key":"shadow%1:15:00::","synset_id":"darkness.n.02"},{"index":8,"sense_key":"shadow%1:26:01::","synset_id":"shadow.n.01"},{"index":9,"sense_key":"shadow%1:09:00::","synset_id":"apparition.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"shadow%1:26:01::","call_ids":["senseval3.d000.s108.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"shadow%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":482,"reasoning_output_tokens":466},"cost":{"currency":"USD","total_usd":0.017605000000000003,"input_uncached_usd":0.0031450000000000002,"input_cached_usd":0.0,"output_usd":0.01446,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.71214229206089}
{"item_id":"senseval3.d000.s108.t003","gold_sense_keys":["make%2:36:00::","make%2:36:08::"],"candidates":[{"index":1,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":2,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":3,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":4,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":5,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":6,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":7,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":8,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":9,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":10,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":11,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":12,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":13,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":14,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":15,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":16,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":17,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":18,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":19,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":20,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":21,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":22,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":23,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":24,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":25,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":26,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":27,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":28,"sense_key":"make%2:29:01::","synset_id":"make.v.49"},{"index":29,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":30,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":31,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":32,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":33,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":34,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":35,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":36,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":37,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":38,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":39,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":40,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":41,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":42,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":43,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":44,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":45,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":46,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":47,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":48,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":49,"sense_key":"make%2:29:06::","synset_id":"make.v.48"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"make%2:36:08::","call_ids":["senseval3.d000.s108.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"make%2:36:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2262,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02715,"input_uncached_usd":0.01131,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.751826624968089}
{"item_id":"senseval3.d000.s108.t005","gold_sense_keys":["noise%1:11:00::"],"candidates":[{"index":1,"sense_key":"noise%1:11:00::","synset_id":"noise.n.01"},{"index":2,"sense_key":"noise%1:07:00::","synset_id":"randomness.n.02"},{"index":3,"sense_key":"noise%1:10:00::","synset_id":"noise.n.04"},{"index":4,"sense_key":"noise%1:11:01::","synset_id":"noise.n.03"},{"index":5,"sense_key":"noise%1:09:00::","synset_id":"noise.n.02"},{"index":6,"sense_key":"noise%1:07:01::","synset_id":"noise.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"noise%1:11:00::","call_ids":["senseval3.d000.s108.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"noise%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":80},"cost":{"currency":"USD","total_usd":0.005795,"input_uncached_usd":0.002945,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1520467499503866}
{"item_id":"senseval3.d000.s108.t006","gold_sense_keys":["shadow%1:09:00::","shadow%1:26:01::"],"candidates":[{"index":1,"sense_key":"shadow%1:10:00::","synset_id":"trace.n.02"},{"index":2,"sense_key":"shadow%1:26:00::","synset_id":"shadow.n.07"},{"index":3,"sense_key":"shadow%1:09:00::","synset_id":"apparition.n.03"},{"index":4,"sense_key":"shadow%1:18:01::","synset_id":"shadow.n.09"},{"index":5,"sense_key":"shadow%1:15:00::","synset_id":"darkness.n.02"},{"index":6,"sense_key":"shadow%1:18:00::","synset_id":"tail.n.05"},{"index":7,"sense_key":"shadow%1:26:01::","synset_id":"shadow.n.01"},{"index":8,"sense_key":"shadow%1:07:00::","synset_id":"shadow.n.06"},{"index":9,"sense_key":"shadow%1:12:00::","synset_id":"shadow.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"shadow%1:26:01::","call_ids":["senseval3.d000.s108.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"shadow%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":285,"reasoning_output_tokens":269},"cost":{"currency":"USD","total_usd":0.011695,"input_uncached_usd":0.0031450000000000002,"input_cached_usd":0.0,"output_usd":0.00855,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.638373832916841}
{"item_id":"senseval3.d000.s109.t001","gold_sense_keys":["shrink%2:30:00::","shrink%2:38:00::"],"candidates":[{"index":1,"sense_key":"shrink%2:30:11::","synset_id":"shrink.v.05"},{"index":2,"sense_key":"shrink%2:30:02::","synset_id":"shrivel.v.01"},{"index":3,"sense_key":"shrink%2:30:00::","synset_id":"shrink.v.04"},{"index":4,"sense_key":"shrink%2:38:00::","synset_id":"flinch.v.01"},{"index":5,"sense_key":"shrink%2:30:01::","synset_id":"shrink.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"shrink%2:38:00::","call_ids":["senseval3.d000.s109.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"shrink%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":95},"cost":{"currency":"USD","total_usd":0.006095,"input_uncached_usd":0.002765,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1247342500137165}
{"item_id":"senseval3.d000.s109.t003","gold_sense_keys":["trap%2:35:00::","trap%2:35:02::","trap%2:41:00::"],"candidates":[{"index":1,"sense_key":"trap%2:35:01::","synset_id":"trap.v.03"},{"index":2,"sense_key":"trap%2:35:02::","synset_id":"trap.v.04"},{"index":3,"sense_key":"trap%2:35:00::","synset_id":"trap.v.02"},{"index":4,"sense_key":"trap%2:41:00::","synset_id":"trap.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trap%2:41:00::","call_ids":["senseval3.d000.s109.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trap%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018245,"input_uncached_usd":0.002405,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.19832179194782}
{"item_id":"senseval3.d000.s109.t004","gold_sense_keys":["corner%1:06:02::"],"candidates":[{"index":1,"sense_key":"corner%1:06:01::","synset_id":"corner.n.11"},{"index":2,"sense_key":"corner%1:15:02::","synset_id":"corner.n.01"},{"index":3,"sense_key":"corner%1:06:00::","synset_id":"corner.n.04"},{"index":4,"sense_key":"corner%1:17:00::","synset_id":"corner.n.09"},{"index":5,"sense_key":"corner%1:15:00::","synset_id":"corner.n.02"},{"index":6,"sense_key":"corner%1:26:01::","synset_id":"corner.n.08"},{"index":7,"sense_key":"corner%1:26:00::","synset_id":"corner.n.07"},{"index":8,"sense_key":"corner%1:06:02::","synset_id":"corner.n.03"},{"index":9,"sense_key":"corner%1:15:01::","synset_id":"corner.n.05"},{"index":10,"sense_key":"corner%1:25:00::","synset_id":"recess.n.02"},{"index":11,"sense_key":"corner%1:15:03::","synset_id":"corner.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"corner%1:06:02::","call_ids":["senseval3.d000.s109.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"corner%1:06:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":709,"cached_input_tokens":0,"output_tokens":408,"reasoning_output_tokens":392},"cost":{"currency":"USD","total_usd":0.015785,"input_uncached_usd":0.0035450000000000004,"input_cached_usd":0.0,"output_usd":0.012240000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.264520332915708}
{"item_id":"senseval3.d000.s109.t005","gold_sense_keys":["wiry%5:00:00:thin:03"],"candidates":[{"index":1,"sense_key":"wiry%3:01:00::","synset_id":"wiry.a.02"},{"index":2,"sense_key":"wiry%5:00:00:thin:03","synset_id":"stringy.s.01"},{"index":3,"sense_key":"wiry%5:00:00:hairy:00","synset_id":"wiry.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"wiry%5:00:00:thin:03","call_ids":["senseval3.d000.s109.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"wiry%5:00:00:thin:03","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":395,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.0038350000000000003,"input_uncached_usd":0.001975,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6502531670266762}
{"item_id":"senseval3.d000.s109.t007","gold_sense_keys":["little%3:00:00::","little%3:00:01::"],"candidates":[{"index":1,"sense_key":"little%3:00:03::","synset_id":"little.a.02"},{"index":2,"sense_key":"little%5:00:00:emotional:00","synset_id":"little.s.05"},{"index":3,"sense_key":"little%3:00:01::","synset_id":"small.a.01"},{"index":4,"sense_key":"little%5:00:00:young:00","synset_id":"little.s.01"},{"index":5,"sense_key":"little%5:00:00:soft:04","synset_id":"little.s.03"},{"index":6,"sense_key":"little%3:00:00::","synset_id":"short.a.03"},{"index":7,"sense_key":"little%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":8,"sense_key":"little%5:00:00:unimportant:00","synset_id":"fiddling.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"little%3:00:00::","call_ids":["senseval3.d000.s109.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"little%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":265,"reasoning_output_tokens":249},"cost":{"currency":"USD","total_usd":0.011425000000000001,"input_uncached_usd":0.0034750000000000002,"input_cached_usd":0.0,"output_usd":0.00795,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.843562708934769}
{"item_id":"senseval3.d000.s109.t008","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":2,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:00::","synset_id":"world.n.08"},{"index":5,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":6,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":7,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":10,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":11,"sense_key":"man%1:18:08::","synset_id":"man.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s109.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":846,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00618,"input_uncached_usd":0.00423,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3451412500580773}
{"item_id":"senseval3.d000.s110.t000","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":2,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"},{"index":3,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":4,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":5,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":6,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":7,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":8,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":9,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":10,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"begin%2:30:00::","call_ids":["senseval3.d000.s110.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"begin%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":831,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":109},"cost":{"currency":"USD","total_usd":0.007905,"input_uncached_usd":0.004155000000000001,"input_cached_usd":0.0,"output_usd":0.0037500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.317379624932073}
{"item_id":"senseval3.d000.s110.t001","gold_sense_keys":["wish%2:32:02::","wish%2:37:00::"],"candidates":[{"index":1,"sense_key":"wish%2:37:00::","synset_id":"wish.v.01"},{"index":2,"sense_key":"wish%2:32:00::","synset_id":"wish.v.06"},{"index":3,"sense_key":"wish%2:32:01::","synset_id":"wish.v.05"},{"index":4,"sense_key":"wish%2:37:02::","synset_id":"wish.v.02"},{"index":5,"sense_key":"wish%2:37:01::","synset_id":"wish.v.04"},{"index":6,"sense_key":"wish%2:32:02::","synset_id":"wish.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wish%2:37:00::","call_ids":["senseval3.d000.s110.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wish%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018485,"input_uncached_usd":0.002645,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":492.49084354098886}
{"item_id":"senseval3.d000.s110.t002","gold_sense_keys":["shout%2:32:01::"],"candidates":[{"index":1,"sense_key":"shout%2:32:01::","synset_id":"shout.v.02"},{"index":2,"sense_key":"shout%2:32:08::","synset_id":"exclaim.v.01"},{"index":3,"sense_key":"shout%2:32:00::","synset_id":"shout.v.01"},{"index":4,"sense_key":"shout%2:32:12::","synset_id":"abuse.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"shout%2:32:01::","call_ids":["senseval3.d000.s110.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"shout%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":446,"reasoning_output_tokens":430},"cost":{"currency":"USD","total_usd":0.01618,"input_uncached_usd":0.0028000000000000004,"input_cached_usd":0.0,"output_usd":0.01338,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.247083749971353}
{"item_id":"senseval3.d000.s110.t003","gold_sense_keys":["other%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":2,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":3,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"other%5:00:00:past:00","call_ids":["senseval3.d000.s110.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"other%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00419,"input_uncached_usd":0.00248,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6530976250069216}
{"item_id":"senseval3.d000.s110.t004","gold_sense_keys":["evening%1:28:00::"],"candidates":[{"index":1,"sense_key":"evening%1:28:02::","synset_id":"evening.n.03"},{"index":2,"sense_key":"evening%1:28:00::","synset_id":"evening.n.01"},{"index":3,"sense_key":"evening%1:28:01::","synset_id":"evening.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"evening%1:28:00::","call_ids":["senseval3.d000.s110.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"evening%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":173,"reasoning_output_tokens":157},"cost":{"currency":"USD","total_usd":0.007465,"input_uncached_usd":0.002275,"input_cached_usd":0.0,"output_usd":0.00519,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8959324999013916}
{"item_id":"senseval3.d000.s110.t005","gold_sense_keys":["truck%1:06:00::"],"candidates":[{"index":1,"sense_key":"truck%1:06:00::","synset_id":"truck.n.01"},{"index":2,"sense_key":"truck%1:06:01::","synset_id":"hand_truck.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"truck%1:06:00::","call_ids":["senseval3.d000.s110.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"truck%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.0035150000000000003,"input_uncached_usd":0.0020150000000000003,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4974420829676092}
{"item_id":"senseval3.d000.s110.t006","gold_sense_keys":["bear_down%2:33:03::"],"candidates":[{"index":1,"sense_key":"bear_down%2:32:12::","synset_id":"bear_down.v.05"},{"index":2,"sense_key":"bear_down%2:35:00::","synset_id":"drag_down.v.01"},{"index":3,"sense_key":"bear_down%2:33:03::","synset_id":"charge.v.01"},{"index":4,"sense_key":"bear_down%2:35:01::","synset_id":"bear_down.v.02"},{"index":5,"sense_key":"bear_down%2:32:01::","synset_id":"bear_down.v.06"},{"index":6,"sense_key":"bear_down%2:33:00::","synset_id":"bear_down.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bear_down%2:33:03::","call_ids":["senseval3.d000.s110.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bear_down%2:33:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":122},"cost":{"currency":"USD","total_usd":0.006925000000000001,"input_uncached_usd":0.002785,"input_cached_usd":0.0,"output_usd":0.0041400000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5834473749855533}
{"item_id":"senseval3.d000.s110.t007","gold_sense_keys":["crossing%1:06:00::","crossing%1:06:01::"],"candidates":[{"index":1,"sense_key":"crossing%1:06:01::","synset_id":"intersection.n.02"},{"index":2,"sense_key":"crossing%1:04:01::","synset_id":"crossing.n.01"},{"index":3,"sense_key":"crossing%1:17:00::","synset_id":"ford.n.07"},{"index":4,"sense_key":"crossing%1:04:00::","synset_id":"crossing.n.07"},{"index":5,"sense_key":"crossing%1:15:00::","synset_id":"crossing.n.03"},{"index":6,"sense_key":"crossing%1:04:02::","synset_id":"hybridization.n.01"},{"index":7,"sense_key":"crossing%1:06:00::","synset_id":"crossing.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"crossing%1:06:00::","call_ids":["senseval3.d000.s110.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"crossing%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018755,"input_uncached_usd":0.002915,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.507225916022435}
{"item_id":"senseval3.d000.s111.t000","gold_sense_keys":["saddle%2:35:10::"],"candidates":[{"index":1,"sense_key":"saddle%2:32:00::","synset_id":"charge.v.18"},{"index":2,"sense_key":"saddle%2:35:10::","synset_id":"saddle.v.02"},{"index":3,"sense_key":"saddle%2:35:01::","synset_id":"saddle.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"saddle%2:35:10::","call_ids":["senseval3.d000.s111.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"saddle%2:35:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.005005000000000001,"input_uncached_usd":0.0020350000000000004,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3857417090330273}
{"item_id":"senseval3.d000.s111.t001","gold_sense_keys":["creep%1:18:00::"],"candidates":[{"index":1,"sense_key":"creep%1:06:00::","synset_id":"creep.n.03"},{"index":2,"sense_key":"creep%1:11:00::","synset_id":"creep.n.02"},{"index":3,"sense_key":"creep%1:18:00::","synset_id":"creep.n.01"},{"index":4,"sense_key":"creep%1:04:00::","synset_id":"crawl.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"creep%1:18:00::","call_ids":["senseval3.d000.s111.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"creep%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.004015,"input_uncached_usd":0.002245,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7243740839185193}
{"item_id":"senseval3.d000.s112.t000","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":2,"sense_key":"start%2:38:03::","synset_id":"start.v.14"},{"index":3,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":4,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":5,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":6,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":7,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":9,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":12,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":13,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":14,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"start%2:30:00::","call_ids":["senseval3.d000.s112.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"start%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":922,"cached_input_tokens":0,"output_tokens":184,"reasoning_output_tokens":168},"cost":{"currency":"USD","total_usd":0.01013,"input_uncached_usd":0.00461,"input_cached_usd":0.0,"output_usd":0.00552,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5157227501040325}
{"item_id":"senseval3.d000.s112.t001","gold_sense_keys":["go%2:38:00::"],"candidates":[{"index":1,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":2,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":3,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"},{"index":4,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":5,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":6,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":7,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":8,"sense_key":"go%2:42:08::","synset_id":"go.v.25"},{"index":9,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":10,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":11,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":12,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"},{"index":13,"sense_key":"go%2:38:01::","synset_id":"go.v.03"},{"index":14,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":15,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":16,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":17,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":18,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":19,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":20,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":21,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":22,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"},{"index":23,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":24,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"},{"index":25,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":26,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":27,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":28,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":29,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":30,"sense_key":"go%2:30:08::","synset_id":"go.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":21,"chosen_sense_key":"go%2:38:00::","call_ids":["senseval3.d000.s112.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":21,"predicted_sense_key":"go%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1693,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":146},"cost":{"currency":"USD","total_usd":0.013325,"input_uncached_usd":0.008465,"input_cached_usd":0.0,"output_usd":0.00486,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.964938541990705}
{"item_id":"senseval3.d000.s112.t002","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":4,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d000.s112.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0042650000000000006,"input_uncached_usd":0.0023450000000000003,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5148641250561923}
{"item_id":"senseval3.d000.s112.t003","gold_sense_keys":["place%1:15:04::"],"candidates":[{"index":1,"sense_key":"place%1:26:00::","synset_id":"place.n.10"},{"index":2,"sense_key":"place%1:10:01::","synset_id":"space.n.07"},{"index":3,"sense_key":"place%1:15:04::","synset_id":"place.n.02"},{"index":4,"sense_key":"place%1:04:01::","synset_id":"stead.n.01"},{"index":5,"sense_key":"place%1:15:00::","synset_id":"topographic_point.n.01"},{"index":6,"sense_key":"place%1:04:00::","synset_id":"position.n.06"},{"index":7,"sense_key":"place%1:26:02::","synset_id":"place.n.13"},{"index":8,"sense_key":"place%1:15:06::","synset_id":"home.n.01"},{"index":9,"sense_key":"place%1:15:05::","synset_id":"position.n.01"},{"index":10,"sense_key":"place%1:09:00::","synset_id":"place.n.03"},{"index":11,"sense_key":"place%1:15:03::","synset_id":"seat.n.01"},{"index":12,"sense_key":"place%1:26:01::","synset_id":"place.n.06"},{"index":13,"sense_key":"place%1:15:02::","synset_id":"plaza.n.01"},{"index":14,"sense_key":"place%1:15:01::","synset_id":"place.n.04"},{"index":15,"sense_key":"place%1:10:02::","synset_id":"place.n.15"},{"index":16,"sense_key":"place%1:10:00::","synset_id":"place.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"place%1:15:04::","call_ids":["senseval3.d000.s112.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"place%1:15:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":971,"cached_input_tokens":0,"output_tokens":214,"reasoning_output_tokens":198},"cost":{"currency":"USD","total_usd":0.011275,"input_uncached_usd":0.004855000000000001,"input_cached_usd":0.0,"output_usd":0.00642,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.87572133296635}
{"item_id":"senseval3.d000.s113.t000","gold_sense_keys":["low%5:00:00:soft:04"],"candidates":[{"index":1,"sense_key":"low%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":2,"sense_key":"low%5:00:00:humble:00","synset_id":"broken.s.01"},{"index":3,"sense_key":"low%3:00:02::","synset_id":"low.a.01"},{"index":4,"sense_key":"low%3:00:03::","synset_id":"low.a.05"},{"index":5,"sense_key":"low%5:00:00:unrefined:01","synset_id":"low.s.02"},{"index":6,"sense_key":"low%5:00:00:contemptible:00","synset_id":"abject.s.01"},{"index":7,"sense_key":"low%5:00:00:dejected:00","synset_id":"gloomy.s.02"},{"index":8,"sense_key":"low%3:00:01::","synset_id":"low.a.02"},{"index":9,"sense_key":"low%5:00:00:soft:04","synset_id":"low.s.01"},{"index":10,"sense_key":"low%5:00:00:insufficient:00","synset_id":"depleted.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"low%5:00:00:soft:04","call_ids":["senseval3.d000.s113.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"low%5:00:00:soft:04","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.005755000000000001,"input_uncached_usd":0.0037450000000000005,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0463630410376936}
{"item_id":"senseval3.d000.s113.t001","gold_sense_keys":["voice%1:07:00::"],"candidates":[{"index":1,"sense_key":"voice%1:10:00::","synset_id":"voice.n.02"},{"index":2,"sense_key":"voice%1:10:03::","synset_id":"voice.n.06"},{"index":3,"sense_key":"voice%1:04:00::","synset_id":"voice.n.05"},{"index":4,"sense_key":"voice%1:07:00::","synset_id":"voice.n.01"},{"index":5,"sense_key":"voice%1:10:01::","synset_id":"articulation.n.03"},{"index":6,"sense_key":"voice%1:07:01::","synset_id":"voice.n.09"},{"index":7,"sense_key":"voice%1:10:02::","synset_id":"part.n.11"},{"index":8,"sense_key":"voice%1:18:00::","synset_id":"spokesperson.n.01"},{"index":9,"sense_key":"voice%1:18:01::","synset_id":"voice.n.07"},{"index":10,"sense_key":"voice%1:11:00::","synset_id":"voice.n.03"},{"index":11,"sense_key":"voice%1:24:00::","synset_id":"voice.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"voice%1:07:00::","call_ids":["senseval3.d000.s113.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"voice%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":782,"cached_input_tokens":0,"output_tokens":353,"reasoning_output_tokens":337},"cost":{"currency":"USD","total_usd":0.0145,"input_uncached_usd":0.00391,"input_cached_usd":0.0,"output_usd":0.01059,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.912818916025572}
{"item_id":"senseval3.d000.s113.t003","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":2,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":3,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":4,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":5,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":6,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":7,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":8,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":9,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":10,"sense_key":"man%1:05:00::","synset_id":"world.n.08"},{"index":11,"sense_key":"man%1:18:08::","synset_id":"man.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s113.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.006475,"input_uncached_usd":0.004195,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8260684579145163}
{"item_id":"senseval3.d000.s113.t004","gold_sense_keys":["ask%2:32:04::"],"candidates":[{"index":1,"sense_key":"ask%2:32:04::","synset_id":"ask.v.05"},{"index":2,"sense_key":"ask%2:32:02::","synset_id":"ask.v.03"},{"index":3,"sense_key":"ask%2:32:01::","synset_id":"ask.v.02"},{"index":4,"sense_key":"ask%2:32:00::","synset_id":"ask.v.01"},{"index":5,"sense_key":"ask%2:32:05::","synset_id":"ask.v.04"},{"index":6,"sense_key":"ask%2:42:00::","synset_id":"necessitate.v.01"},{"index":7,"sense_key":"ask%2:32:09::","synset_id":"ask.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ask%2:32:04::","call_ids":["senseval3.d000.s113.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ask%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":129},"cost":{"currency":"USD","total_usd":0.007390000000000001,"input_uncached_usd":0.0030700000000000002,"input_cached_usd":0.0,"output_usd":0.00432,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.130659291986376}
{"item_id":"senseval3.d000.s113.t005","gold_sense_keys":["happy%3:00:00::"],"candidates":[{"index":1,"sense_key":"happy%3:00:00::","synset_id":"happy.a.01"},{"index":2,"sense_key":"happy%5:00:00:willing:00","synset_id":"glad.s.01"},{"index":3,"sense_key":"happy%5:00:00:felicitous:00","synset_id":"happy.s.03"},{"index":4,"sense_key":"happy%5:00:00:fortunate:00","synset_id":"felicitous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"happy%3:00:00::","call_ids":["senseval3.d000.s113.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"happy%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":115},"cost":{"currency":"USD","total_usd":0.00626,"input_uncached_usd":0.00236,"input_cached_usd":0.0,"output_usd":0.0039000000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1359917499357834}
{"item_id":"senseval3.d000.s113.t006","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:21:00::","synset_id":"way.n.12"},{"index":2,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":3,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":6,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":7,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":8,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":9,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":10,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":11,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":12,"sense_key":"way%1:23:00::","synset_id":"room.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"way%1:07:01::","call_ids":["senseval3.d000.s113.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":98},"cost":{"currency":"USD","total_usd":0.007285,"input_uncached_usd":0.0038650000000000004,"input_cached_usd":0.0,"output_usd":0.0034200000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6924542920896783}
{"item_id":"senseval3.d000.s114.t000","gold_sense_keys":["make%2:41:13::"],"candidates":[{"index":1,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":4,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":5,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":6,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":7,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":8,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":9,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":10,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":11,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":12,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":13,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":16,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":17,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":18,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":19,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":20,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":21,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":22,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":23,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":24,"sense_key":"make%2:29:01::","synset_id":"make.v.49"},{"index":25,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":26,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":27,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":28,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":29,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":30,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":31,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":32,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":33,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":34,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":35,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":36,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":37,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":38,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":39,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":40,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":41,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":42,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":43,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":44,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":45,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":46,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":47,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"make%2:41:13::","call_ids":["senseval3.d000.s114.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"make%2:41:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2252,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.013600000000000001,"input_uncached_usd":0.011260000000000001,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.588248666957952}
{"item_id":"senseval3.d000.s114.t001","gold_sense_keys":["mistake%1:04:00::"],"candidates":[{"index":1,"sense_key":"mistake%1:10:00::","synset_id":"error.n.07"},{"index":2,"sense_key":"mistake%1:09:00::","synset_id":"mistake.n.02"},{"index":3,"sense_key":"mistake%1:04:00::","synset_id":"mistake.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"mistake%1:04:00::","call_ids":["senseval3.d000.s114.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"mistake%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.003915,"input_uncached_usd":0.0022050000000000004,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5457909590331838}
{"item_id":"senseval3.d000.s114.t002","gold_sense_keys":["answer%2:32:00::"],"candidates":[{"index":1,"sense_key":"answer%2:34:00::","synset_id":"answer.v.09"},{"index":2,"sense_key":"answer%2:31:02::","synset_id":"answer.v.02"},{"index":3,"sense_key":"answer%2:31:05::","synset_id":"answer.v.10"},{"index":4,"sense_key":"answer%2:32:00::","synset_id":"answer.v.01"},{"index":5,"sense_key":"answer%2:32:06::","synset_id":"answer.v.05"},{"index":6,"sense_key":"answer%2:42:00::","synset_id":"answer.v.08"},{"index":7,"sense_key":"answer%2:31:03::","synset_id":"answer.v.04"},{"index":8,"sense_key":"answer%2:42:01::","synset_id":"answer.v.06"},{"index":9,"sense_key":"answer%2:42:02::","synset_id":"suffice.v.01"},{"index":10,"sense_key":"answer%2:31:00::","synset_id":"answer.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"answer%2:32:00::","call_ids":["senseval3.d000.s114.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"answer%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":676,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.005810000000000001,"input_uncached_usd":0.00338,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.450603708042763}
{"item_id":"senseval3.d000.s114.t004","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":2,"sense_key":"way%1:21:00::","synset_id":"way.n.12"},{"index":3,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":4,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":5,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":6,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":7,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":8,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":9,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":10,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":11,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":12,"sense_key":"way%1:06:00::","synset_id":"way.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"way%1:07:01::","call_ids":["senseval3.d000.s114.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.0062250000000000005,"input_uncached_usd":0.0038250000000000003,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7697042090585455}
{"item_id":"senseval3.d000.s114.t005","gold_sense_keys":["realize%2:31:00::"],"candidates":[{"index":1,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"},{"index":2,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"},{"index":3,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"},{"index":4,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":5,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"},{"index":6,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"realize%2:31:00::","call_ids":["senseval3.d000.s114.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"realize%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018795,"input_uncached_usd":0.002955,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.564625750062987}
{"item_id":"senseval3.d000.s114.t006","gold_sense_keys":["skepticism%1:09:00::"],"candidates":[{"index":1,"sense_key":"skepticism%1:09:00::","synset_id":"incredulity.n.01"},{"index":2,"sense_key":"skepticism%1:09:01::","synset_id":"agnosticism.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"skepticism%1:09:00::","call_ids":["senseval3.d000.s114.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"skepticism%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":369,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.004155,"input_uncached_usd":0.001845,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.208976124995388}
{"item_id":"senseval3.d000.s114.t007","gold_sense_keys":["show%2:39:01::"],"candidates":[{"index":1,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":2,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":5,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":6,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":7,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":8,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":9,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":10,"sense_key":"show%2:33:00::","synset_id":"show.v.12"},{"index":11,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":12,"sense_key":"show%2:32:02::","synset_id":"express.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"show%2:39:01::","call_ids":["senseval3.d000.s114.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"show%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":866,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":146},"cost":{"currency":"USD","total_usd":0.00919,"input_uncached_usd":0.0043300000000000005,"input_cached_usd":0.0,"output_usd":0.00486,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.75843370903749}
{"item_id":"senseval3.d000.s114.t008","gold_sense_keys":["face%1:07:03::","face%1:08:00::"],"candidates":[{"index":1,"sense_key":"face%1:18:00::","synset_id":"face.n.05"},{"index":2,"sense_key":"face%1:08:00::","synset_id":"face.n.01"},{"index":3,"sense_key":"face%1:07:01::","synset_id":"face.n.11"},{"index":4,"sense_key":"face%1:07:03::","synset_id":"expression.n.01"},{"index":5,"sense_key":"face%1:10:00::","synset_id":"font.n.01"},{"index":6,"sense_key":"face%1:06:01::","synset_id":"face.n.08"},{"index":7,"sense_key":"face%1:07:00::","synset_id":"face.n.03"},{"index":8,"sense_key":"face%1:07:02::","synset_id":"boldness.n.02"},{"index":9,"sense_key":"face%1:15:00::","synset_id":"side.n.04"},{"index":10,"sense_key":"face%1:08:01::","synset_id":"face.n.07"},{"index":11,"sense_key":"face%1:10:01::","synset_id":"grimace.n.01"},{"index":12,"sense_key":"face%1:06:00::","synset_id":"face.n.13"},{"index":13,"sense_key":"face%1:06:02::","synset_id":"face.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"face%1:07:03::","call_ids":["senseval3.d000.s114.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"face%1:07:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":814,"cached_input_tokens":0,"output_tokens":170,"reasoning_output_tokens":154},"cost":{"currency":"USD","total_usd":0.009170000000000001,"input_uncached_usd":0.004070000000000001,"input_cached_usd":0.0,"output_usd":0.0051,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1861433748854324}
{"item_id":"senseval3.d000.s114.t009","gold_sense_keys":["voice%1:07:00::"],"candidates":[{"index":1,"sense_key":"voice%1:07:01::","synset_id":"voice.n.09"},{"index":2,"sense_key":"voice%1:10:01::","synset_id":"articulation.n.03"},{"index":3,"sense_key":"voice%1:10:00::","synset_id":"voice.n.02"},{"index":4,"sense_key":"voice%1:10:02::","synset_id":"part.n.11"},{"index":5,"sense_key":"voice%1:10:03::","synset_id":"voice.n.06"},{"index":6,"sense_key":"voice%1:07:00::","synset_id":"voice.n.01"},{"index":7,"sense_key":"voice%1:11:00::","synset_id":"voice.n.03"},{"index":8,"sense_key":"voice%1:04:00::","synset_id":"voice.n.05"},{"index":9,"sense_key":"voice%1:18:00::","synset_id":"spokesperson.n.01"},{"index":10,"sense_key":"voice%1:18:01::","synset_id":"voice.n.07"},{"index":11,"sense_key":"voice%1:24:00::","synset_id":"voice.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"voice%1:07:00::","call_ids":["senseval3.d000.s114.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"voice%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":775,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":97},"cost":{"currency":"USD","total_usd":0.007265000000000001,"input_uncached_usd":0.0038750000000000004,"input_cached_usd":0.0,"output_usd":0.0033900000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.291468624956906}
{"item_id":"senseval3.d000.s115.t001","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":2,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"become%2:30:00::","call_ids":["senseval3.d000.s115.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"become%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.003915,"input_uncached_usd":0.002295,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6262605000520125}
{"item_id":"senseval3.d000.s115.t002","gold_sense_keys":["sad%3:00:00::"],"candidates":[{"index":1,"sense_key":"sad%3:00:00::","synset_id":"sad.a.01"},{"index":2,"sense_key":"sad%5:00:00:sorrowful:00","synset_id":"sad.s.01"},{"index":3,"sense_key":"sad%5:00:00:bad:00","synset_id":"deplorable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sad%3:00:00::","call_ids":["senseval3.d000.s115.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sad%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.0038600000000000006,"input_uncached_usd":0.0023000000000000004,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.469083167030476}
{"item_id":"senseval3.d000.s115.t003","gold_sense_keys":["moody%5:00:00:ill-natured:00"],"candidates":[{"index":1,"sense_key":"moody%5:00:00:emotional:00","synset_id":"moody.s.02"},{"index":2,"sense_key":"moody%5:00:00:ill-natured:00","synset_id":"dark.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"moody%5:00:00:ill-natured:00","call_ids":["senseval3.d000.s115.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"moody%5:00:00:ill-natured:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":108},"cost":{"currency":"USD","total_usd":0.005765000000000001,"input_uncached_usd":0.0020450000000000004,"input_cached_usd":0.0,"output_usd":0.00372,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.28952658304479}
{"item_id":"senseval3.d000.s116.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":3,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":4,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["senseval3.d000.s116.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.005445,"input_uncached_usd":0.0023850000000000004,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.51512749993708}
{"item_id":"senseval3.d000.s116.t002","gold_sense_keys":["company%1:26:00::"],"candidates":[{"index":1,"sense_key":"company%1:14:05::","synset_id":"company.n.09"},{"index":2,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":3,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":4,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":5,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":6,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":7,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":8,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":9,"sense_key":"company%1:14:01::","synset_id":"company.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"company%1:26:00::","call_ids":["senseval3.d000.s116.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"company%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":98},"cost":{"currency":"USD","total_usd":0.006575,"input_uncached_usd":0.0031550000000000003,"input_cached_usd":0.0,"output_usd":0.0034200000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3230675410013646}
{"item_id":"senseval3.d000.s116.t003","gold_sense_keys":["feel%2:31:00::"],"candidates":[{"index":1,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":2,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":3,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":4,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":5,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":6,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":7,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"},{"index":8,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":9,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":10,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":11,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":12,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":13,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"feel%2:31:00::","call_ids":["senseval3.d000.s116.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"feel%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":825,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.007335,"input_uncached_usd":0.004125,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.209597499924712}
{"item_id":"senseval3.d000.s116.t004","gold_sense_keys":["buy%2:40:00::"],"candidates":[{"index":1,"sense_key":"buy%2:42:00::","synset_id":"buy.v.03"},{"index":2,"sense_key":"buy%2:40:02::","synset_id":"bribe.v.01"},{"index":3,"sense_key":"buy%2:31:09::","synset_id":"buy.v.05"},{"index":4,"sense_key":"buy%2:40:03::","synset_id":"buy.v.04"},{"index":5,"sense_key":"buy%2:40:00::","synset_id":"buy.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"buy%2:40:00::","call_ids":["senseval3.d000.s116.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"buy%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.004335,"input_uncached_usd":0.002415,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6928360000019893}
{"item_id":"senseval3.d000.s116.t005","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":2,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":3,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"},{"index":4,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":5,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s116.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.004395,"input_uncached_usd":0.002475,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7593823749339208}
{"item_id":"senseval3.d000.s116.t006","gold_sense_keys":["prevent%2:41:01::"],"candidates":[{"index":1,"sense_key":"prevent%2:41:01::","synset_id":"prevent.v.01"},{"index":2,"sense_key":"prevent%2:41:00::","synset_id":"prevent.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"prevent%2:41:01::","call_ids":["senseval3.d000.s116.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"prevent%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":399,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.005505,"input_uncached_usd":0.0019950000000000002,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.892540749977343}
{"item_id":"senseval3.d000.s116.t007","gold_sense_keys":["possible%3:00:04::"],"candidates":[{"index":1,"sense_key":"possible%3:00:04::","synset_id":"potential.a.01"},{"index":2,"sense_key":"possible%3:00:00::","synset_id":"possible.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"possible%3:00:04::","call_ids":["senseval3.d000.s116.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"possible%3:00:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":351,"cached_input_tokens":0,"output_tokens":161,"reasoning_output_tokens":146},"cost":{"currency":"USD","total_usd":0.006585000000000001,"input_uncached_usd":0.001755,"input_cached_usd":0.0,"output_usd":0.00483,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.90710545796901}
{"item_id":"senseval3.d000.s116.t008","gold_sense_keys":["trouble%1:04:02::","trouble%1:09:00::","trouble%1:11:00::","trouble%1:11:01::"],"candidates":[{"index":1,"sense_key":"trouble%1:04:02::","synset_id":"trouble.n.04"},{"index":2,"sense_key":"trouble%1:12:03::","synset_id":"worry.n.02"},{"index":3,"sense_key":"trouble%1:26:02::","synset_id":"trouble.n.06"},{"index":4,"sense_key":"trouble%1:11:01::","synset_id":"fuss.n.02"},{"index":5,"sense_key":"trouble%1:11:00::","synset_id":"trouble.n.03"},{"index":6,"sense_key":"trouble%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trouble%1:11:01::","call_ids":["senseval3.d000.s116.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trouble%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":115},"cost":{"currency":"USD","total_usd":0.006515,"input_uncached_usd":0.002585,"input_cached_usd":0.0,"output_usd":0.00393,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8268750000279397}
{"item_id":"senseval3.d000.s117.t000","gold_sense_keys":["trouble%1:04:02::","trouble%1:09:00::","trouble%1:11:00::","trouble%1:11:01::"],"candidates":[{"index":1,"sense_key":"trouble%1:11:01::","synset_id":"fuss.n.02"},{"index":2,"sense_key":"trouble%1:04:02::","synset_id":"trouble.n.04"},{"index":3,"sense_key":"trouble%1:12:03::","synset_id":"worry.n.02"},{"index":4,"sense_key":"trouble%1:26:02::","synset_id":"trouble.n.06"},{"index":5,"sense_key":"trouble%1:09:00::","synset_id":"trouble.n.01"},{"index":6,"sense_key":"trouble%1:11:00::","synset_id":"trouble.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trouble%1:11:01::","call_ids":["senseval3.d000.s117.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trouble%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":139},"cost":{"currency":"USD","total_usd":0.0072,"input_uncached_usd":0.0025800000000000003,"input_cached_usd":0.0,"output_usd":0.00462,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.385649583069608}
{"item_id":"senseval3.d000.s118.t000","gold_sense_keys":["guy%1:18:00::"],"candidates":[{"index":1,"sense_key":"guy%1:06:01::","synset_id":"guy.n.02"},{"index":2,"sense_key":"guy%1:06:00::","synset_id":"guy.n.03"},{"index":3,"sense_key":"guy%1:18:00::","synset_id":"guy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"guy%1:18:00::","call_ids":["senseval3.d000.s118.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"guy%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.0034000000000000002,"input_uncached_usd":0.00205,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9023635409539565}
{"item_id":"senseval3.d000.s118.t002","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":2,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":3,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"},{"index":4,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":5,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s118.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.005245,"input_uncached_usd":0.002455,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7846782910637558}
{"item_id":"senseval3.d000.s118.t003","gold_sense_keys":["upset%2:37:00::","upset%2:37:01::"],"candidates":[{"index":1,"sense_key":"upset%2:35:00::","synset_id":"swage.v.01"},{"index":2,"sense_key":"upset%2:33:00::","synset_id":"upset.v.06"},{"index":3,"sense_key":"upset%2:30:00::","synset_id":"upset.v.01"},{"index":4,"sense_key":"upset%2:37:00::","synset_id":"disturb.v.01"},{"index":5,"sense_key":"upset%2:37:01::","synset_id":"upset.v.02"},{"index":6,"sense_key":"upset%2:38:00::","synset_id":"overturn.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"upset%2:37:00::","call_ids":["senseval3.d000.s118.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"upset%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01845,"input_uncached_usd":0.0026100000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.706907583982684}
{"item_id":"senseval3.d000.s118.t005","gold_sense_keys":["appreciation%1:10:00::"],"candidates":[{"index":1,"sense_key":"appreciation%1:04:01::","synset_id":"appreciation.n.05"},{"index":2,"sense_key":"appreciation%1:10:00::","synset_id":"appreciation.n.03"},{"index":3,"sense_key":"appreciation%1:09:00::","synset_id":"appreciation.n.01"},{"index":4,"sense_key":"appreciation%1:04:00::","synset_id":"admiration.n.03"},{"index":5,"sense_key":"appreciation%1:09:01::","synset_id":"taste.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"appreciation%1:10:00::","call_ids":["senseval3.d000.s118.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"appreciation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01836,"input_uncached_usd":0.00252,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.88301341701299}
{"item_id":"senseval3.d000.s119.t000","gold_sense_keys":["break%2:30:03::","break%2:42:05::"],"candidates":[{"index":1,"sense_key":"break%2:40:07::","synset_id":"bankrupt.v.01"},{"index":2,"sense_key":"break%2:38:08::","synset_id":"break.v.08"},{"index":3,"sense_key":"break%2:41:00::","synset_id":"transgress.v.01"},{"index":4,"sense_key":"break%2:30:03::","synset_id":"interrupt.v.04"},{"index":5,"sense_key":"break%2:35:00::","synset_id":"break.v.42"},{"index":6,"sense_key":"break%2:35:05::","synset_id":"break.v.43"},{"index":7,"sense_key":"break%2:38:01::","synset_id":"break.v.20"},{"index":8,"sense_key":"break%2:38:15::","synset_id":"break.v.34"},{"index":9,"sense_key":"break%2:42:03::","synset_id":"break.v.26"},{"index":10,"sense_key":"break%2:41:09::","synset_id":"break.v.29"},{"index":11,"sense_key":"break%2:33:00::","synset_id":"better.v.01"},{"index":12,"sense_key":"break%2:35:10::","synset_id":"break.v.39"},{"index":13,"sense_key":"break%2:35:14::","synset_id":"break.v.40"},{"index":14,"sense_key":"break%2:32:04::","synset_id":"pause.v.02"},{"index":15,"sense_key":"break%2:32:00::","synset_id":"unwrap.v.02"},{"index":16,"sense_key":"break%2:38:11::","synset_id":"collapse.v.01"},{"index":17,"sense_key":"break%2:42:15::","synset_id":"break.v.23"},{"index":18,"sense_key":"break%2:41:15::","synset_id":"break.v.28"},{"index":19,"sense_key":"break%2:38:09::","synset_id":"break.v.35"},{"index":20,"sense_key":"break%2:31:06::","synset_id":"break.v.50"},{"index":21,"sense_key":"break%2:30:10::","synset_id":"break.v.05"},{"index":22,"sense_key":"break%2:41:06::","synset_id":"demote.v.01"},{"index":23,"sense_key":"break%2:35:13::","synset_id":"break.v.44"},{"index":24,"sense_key":"break%2:30:11::","synset_id":"break_in.v.06"},{"index":25,"sense_key":"break%2:42:14::","synset_id":"break.v.27"},{"index":26,"sense_key":"break%2:30:14::","synset_id":"break.v.52"},{"index":27,"sense_key":"break%2:38:00::","synset_id":"break_dance.v.01"},{"index":28,"sense_key":"break%2:30:06::","synset_id":"break.v.03"},{"index":29,"sense_key":"break%2:32:13::","synset_id":"break.v.48"},{"index":30,"sense_key":"break%2:30:05::","synset_id":"fail.v.04"},{"index":31,"sense_key":"break%2:30:00::","synset_id":"break.v.02"},{"index":32,"sense_key":"break%2:29:10::","synset_id":"break.v.59"},{"index":33,"sense_key":"break%2:35:01::","synset_id":"break.v.04"},{"index":34,"sense_key":"break%2:41:01::","synset_id":"break.v.18"},{"index":35,"sense_key":"break%2:38:10::","synset_id":"break.v.19"},{"index":36,"sense_key":"break%2:35:04::","synset_id":"break.v.41"},{"index":37,"sense_key":"break%2:30:09::","synset_id":"dampen.v.07"},{"index":38,"sense_key":"break%2:35:11::","synset_id":"break.v.38"},{"index":39,"sense_key":"break%2:41:13::","synset_id":"separate.v.08"},{"index":40,"sense_key":"break%2:32:07::","synset_id":"break.v.49"},{"index":41,"sense_key":"break%2:42:05::","synset_id":"break.v.24"},{"index":42,"sense_key":"break%2:42:04::","synset_id":"break.v.25"},{"index":43,"sense_key":"break%2:30:04::","synset_id":"break.v.16"},{"index":44,"sense_key":"break%2:42:00::","synset_id":"violate.v.01"},{"index":45,"sense_key":"break%2:30:01::","synset_id":"break.v.53"},{"index":46,"sense_key":"break%2:41:10::","synset_id":"break.v.30"},{"index":47,"sense_key":"break%2:30:15::","synset_id":"crack.v.01"},{"index":48,"sense_key":"break%2:32:05::","synset_id":"break.v.46"},{"index":49,"sense_key":"break%2:41:11::","synset_id":"break_in.v.01"},{"index":50,"sense_key":"break%2:30:07::","synset_id":"break.v.55"},{"index":51,"sense_key":"break%2:37:00::","synset_id":"break.v.09"},{"index":52,"sense_key":"break%2:31:05::","synset_id":"break.v.51"},{"index":53,"sense_key":"break%2:29:04::","synset_id":"fracture.v.06"},{"index":54,"sense_key":"break%2:30:12::","synset_id":"break.v.22"},{"index":55,"sense_key":"break%2:30:02::","synset_id":"break.v.10"},{"index":56,"sense_key":"break%2:29:11::","synset_id":"break.v.58"},{"index":57,"sense_key":"break%2:35:09::","synset_id":"break.v.45"},{"index":58,"sense_key":"break%2:38:02::","synset_id":"break.v.07"},{"index":59,"sense_key":"break%2:30:08::","synset_id":"break.v.56"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"break%2:30:03::","call_ids":["senseval3.d000.s119.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"break%2:30:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2668,"cached_input_tokens":0,"output_tokens":898,"reasoning_output_tokens":877},"cost":{"currency":"USD","total_usd":0.040279999999999996,"input_uncached_usd":0.013340000000000001,"input_cached_usd":0.0,"output_usd":0.026940000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.88960841705557}
{"item_id":"senseval3.d000.s119.t001","gold_sense_keys":["uncomfortable%3:00:01::"],"candidates":[{"index":1,"sense_key":"uncomfortable%3:00:01::","synset_id":"uncomfortable.a.01"},{"index":2,"sense_key":"uncomfortable%3:00:00::","synset_id":"uncomfortable.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"uncomfortable%3:00:01::","call_ids":["senseval3.d000.s119.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"uncomfortable%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":376,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.0035900000000000003,"input_uncached_usd":0.0018800000000000002,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3123027919791639}
{"item_id":"senseval3.d000.s119.t002","gold_sense_keys":["silence%1:26:00::"],"candidates":[{"index":1,"sense_key":"silence%1:07:00::","synset_id":"silence.n.02"},{"index":2,"sense_key":"silence%1:07:02::","synset_id":"secrecy.n.01"},{"index":3,"sense_key":"silence%1:26:00::","synset_id":"silence.n.01"},{"index":4,"sense_key":"silence%1:07:01::","synset_id":"muteness.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"silence%1:26:00::","call_ids":["senseval3.d000.s119.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"silence%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.004905,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.00264,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4981633339775726}
{"item_id":"senseval3.d000.s119.t003","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":2,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":4,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":6,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":7,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"},{"index":8,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":9,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":10,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"begin%2:30:00::","call_ids":["senseval3.d000.s119.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"begin%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":805,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019865,"input_uncached_usd":0.004025,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.829570375033654}
{"item_id":"senseval3.d000.s120.t000","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":2,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":3,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":4,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":5,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s120.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.005465,"input_uncached_usd":0.0024950000000000003,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1776316249743104}
{"item_id":"senseval3.d000.s120.t001","gold_sense_keys":["complain%2:32:00::"],"candidates":[{"index":1,"sense_key":"complain%2:32:00::","synset_id":"complain.v.01"},{"index":2,"sense_key":"complain%2:32:01::","synset_id":"complain.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"complain%2:32:00::","call_ids":["senseval3.d000.s120.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"complain%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":388,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.00341,"input_uncached_usd":0.00194,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1132852089358494}
{"item_id":"senseval3.d000.s120.t003","gold_sense_keys":["subject%1:09:01::","subject%1:10:00::"],"candidates":[{"index":1,"sense_key":"subject%1:09:00::","synset_id":"discipline.n.01"},{"index":2,"sense_key":"subject%1:06:00::","synset_id":"subject.n.02"},{"index":3,"sense_key":"subject%1:18:00::","synset_id":"national.n.01"},{"index":4,"sense_key":"subject%1:10:01::","synset_id":"subject.n.05"},{"index":5,"sense_key":"subject%1:10:00::","synset_id":"subject.n.01"},{"index":6,"sense_key":"subject%1:18:01::","synset_id":"subject.n.06"},{"index":7,"sense_key":"subject%1:10:02::","synset_id":"subject.n.08"},{"index":8,"sense_key":"subject%1:09:01::","synset_id":"topic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"subject%1:10:00::","call_ids":["senseval3.d000.s120.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"subject%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.0056500000000000005,"input_uncached_usd":0.00334,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.882930124993436}
{"item_id":"senseval3.d000.s120.t004","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":2,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":3,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":4,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"},{"index":5,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":6,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"minute%1:28:00::","call_ids":["senseval3.d000.s120.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.004705000000000001,"input_uncached_usd":0.002635,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.84389841707889}
{"item_id":"senseval3.d000.s120.t005","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":2,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":3,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"},{"index":4,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d000.s120.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01826,"input_uncached_usd":0.0024200000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.86848450009711}
{"item_id":"senseval3.d000.s120.t007","gold_sense_keys":["interest%1:09:00::"],"candidates":[{"index":1,"sense_key":"interest%1:07:01::","synset_id":"sake.n.01"},{"index":2,"sense_key":"interest%1:07:02::","synset_id":"interest.n.03"},{"index":3,"sense_key":"interest%1:14:00::","synset_id":"interest.n.06"},{"index":4,"sense_key":"interest%1:21:03::","synset_id":"interest.n.05"},{"index":5,"sense_key":"interest%1:04:01::","synset_id":"pastime.n.01"},{"index":6,"sense_key":"interest%1:21:00::","synset_id":"interest.n.04"},{"index":7,"sense_key":"interest%1:09:00::","synset_id":"interest.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"interest%1:09:00::","call_ids":["senseval3.d000.s120.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"interest%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":110},"cost":{"currency":"USD","total_usd":0.006840000000000001,"input_uncached_usd":0.0030600000000000002,"input_cached_usd":0.0,"output_usd":0.00378,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.923822124954313}
{"item_id":"senseval3.d000.s120.t009","gold_sense_keys":["show%2:39:01::"],"candidates":[{"index":1,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":2,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":3,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":4,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":5,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":6,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":7,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":8,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":9,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":10,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":11,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":12,"sense_key":"show%2:33:00::","synset_id":"show.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"show%2:39:01::","call_ids":["senseval3.d000.s120.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"show%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":854,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":131},"cost":{"currency":"USD","total_usd":0.00868,"input_uncached_usd":0.00427,"input_cached_usd":0.0,"output_usd":0.00441,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7892824170412496}
{"item_id":"senseval3.d000.s120.t010","gold_sense_keys":["alert%3:00:00::","alert%5:00:00:aware:00"],"candidates":[{"index":1,"sense_key":"alert%5:00:00:aware:00","synset_id":"alert.s.02"},{"index":2,"sense_key":"alert%3:00:00::","synset_id":"alert.a.01"},{"index":3,"sense_key":"alert%5:00:00:energetic:00","synset_id":"alert.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"alert%5:00:00:aware:00","call_ids":["senseval3.d000.s120.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"alert%5:00:00:aware:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":429,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":110},"cost":{"currency":"USD","total_usd":0.005925,"input_uncached_usd":0.002145,"input_cached_usd":0.0,"output_usd":0.00378,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6418184170033783}
{"item_id":"senseval3.d000.s120.t011","gold_sense_keys":["face%1:07:03::"],"candidates":[{"index":1,"sense_key":"face%1:07:00::","synset_id":"face.n.03"},{"index":2,"sense_key":"face%1:18:00::","synset_id":"face.n.05"},{"index":3,"sense_key":"face%1:08:00::","synset_id":"face.n.01"},{"index":4,"sense_key":"face%1:06:02::","synset_id":"face.n.04"},{"index":5,"sense_key":"face%1:10:01::","synset_id":"grimace.n.01"},{"index":6,"sense_key":"face%1:07:01::","synset_id":"face.n.11"},{"index":7,"sense_key":"face%1:15:00::","synset_id":"side.n.04"},{"index":8,"sense_key":"face%1:07:03::","synset_id":"expression.n.01"},{"index":9,"sense_key":"face%1:08:01::","synset_id":"face.n.07"},{"index":10,"sense_key":"face%1:07:02::","synset_id":"boldness.n.02"},{"index":11,"sense_key":"face%1:06:00::","synset_id":"face.n.13"},{"index":12,"sense_key":"face%1:06:01::","synset_id":"face.n.08"},{"index":13,"sense_key":"face%1:10:00::","synset_id":"font.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"face%1:07:03::","call_ids":["senseval3.d000.s120.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"face%1:07:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":802,"cached_input_tokens":0,"output_tokens":177,"reasoning_output_tokens":161},"cost":{"currency":"USD","total_usd":0.009320000000000002,"input_uncached_usd":0.0040100000000000005,"input_cached_usd":0.0,"output_usd":0.0053100000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.077451540972106}
{"item_id":"senseval3.d000.s120.t012","gold_sense_keys":["bright%5:00:00:happy:00"],"candidates":[{"index":1,"sense_key":"bright%5:00:00:happy:00","synset_id":"bright.s.07"},{"index":2,"sense_key":"bright%3:00:02::","synset_id":"undimmed.a.01"},{"index":3,"sense_key":"bright%5:00:00:reverberant:00","synset_id":"bright.s.06"},{"index":4,"sense_key":"bright%5:00:00:colorful:00","synset_id":"bright.s.01"},{"index":5,"sense_key":"bright%3:00:00::","synset_id":"bright.a.01"},{"index":6,"sense_key":"bright%5:00:00:auspicious:00","synset_id":"bright.s.08"},{"index":7,"sense_key":"bright%5:00:00:intelligent:00","synset_id":"bright.s.02"},{"index":8,"sense_key":"bright%5:00:00:glorious:00","synset_id":"bright.s.05"},{"index":9,"sense_key":"bright%5:00:00:polished:00","synset_id":"bright.s.04"},{"index":10,"sense_key":"bright%5:00:00:light:06","synset_id":"bright.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"bright%3:00:00::","call_ids":["senseval3.d000.s120.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"bright%3:00:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01965,"input_uncached_usd":0.0038100000000000005,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.925395584083162}
{"item_id":"senseval3.d000.s120.t013","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"},{"index":2,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":5,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s120.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.003985,"input_uncached_usd":0.002395,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3584586250362918}
{"item_id":"senseval3.d000.s121.t000","gold_sense_keys":["pause%2:42:00::"],"candidates":[{"index":1,"sense_key":"pause%2:42:00::","synset_id":"hesitate.v.02"},{"index":2,"sense_key":"pause%2:32:01::","synset_id":"pause.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pause%2:42:00::","call_ids":["senseval3.d000.s121.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pause%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":355,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":108},"cost":{"currency":"USD","total_usd":0.005465,"input_uncached_usd":0.001775,"input_cached_usd":0.0,"output_usd":0.00369,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1247135000303388}
{"item_id":"senseval3.d000.s121.t001","gold_sense_keys":["moisten%2:30:00::"],"candidates":[{"index":1,"sense_key":"moisten%2:30:00::","synset_id":"moisten.v.01"},{"index":2,"sense_key":"moisten%2:35:00::","synset_id":"drizzle.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"moisten%2:30:00::","call_ids":["senseval3.d000.s121.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"moisten%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":357,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.003765,"input_uncached_usd":0.0017850000000000001,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9744819999905303}
{"item_id":"senseval3.d000.s121.t002","gold_sense_keys":["throat%1:08:00::"],"candidates":[{"index":1,"sense_key":"throat%1:06:01::","synset_id":"throat.n.02"},{"index":2,"sense_key":"throat%1:06:00::","synset_id":"throat.n.03"},{"index":3,"sense_key":"throat%1:05:00::","synset_id":"throat.n.04"},{"index":4,"sense_key":"throat%1:08:00::","synset_id":"throat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"throat%1:08:00::","call_ids":["senseval3.d000.s121.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"throat%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.003725,"input_uncached_usd":0.002135,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6512929999735206}
{"item_id":"senseval3.d000.s121.t003","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s121.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":342,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":96},"cost":{"currency":"USD","total_usd":0.00507,"input_uncached_usd":0.0017100000000000001,"input_cached_usd":0.0,"output_usd":0.00336,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.205890541896224}
{"item_id":"senseval3.d000.s121.t004","gold_sense_keys":["break_in%2:32:00::"],"candidates":[{"index":1,"sense_key":"break_in%2:30:00::","synset_id":"break_in.v.06"},{"index":2,"sense_key":"break_in%2:41:00::","synset_id":"break_in.v.01"},{"index":3,"sense_key":"break_in%2:30:01::","synset_id":"break_in.v.05"},{"index":4,"sense_key":"break_in%2:35:00::","synset_id":"break_in.v.04"},{"index":5,"sense_key":"break_in%2:30:03::","synset_id":"break_in.v.03"},{"index":6,"sense_key":"break_in%2:32:00::","synset_id":"chime_in.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"break_in%2:32:00::","call_ids":["senseval3.d000.s121.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"break_in%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0046500000000000005,"input_uncached_usd":0.0027600000000000003,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6811539579648525}
{"item_id":"senseval3.d000.s122.t000","gold_sense_keys":["pay%2:40:14::"],"candidates":[{"index":1,"sense_key":"pay%2:40:04::","synset_id":"pay_up.v.01"},{"index":2,"sense_key":"pay%2:32:00::","synset_id":"give.v.05"},{"index":3,"sense_key":"pay%2:40:14::","synset_id":"pay.v.11"},{"index":4,"sense_key":"pay%2:40:00::","synset_id":"pay.v.01"},{"index":5,"sense_key":"pay%2:40:02::","synset_id":"pay.v.05"},{"index":6,"sense_key":"pay%2:41:00::","synset_id":"pay.v.08"},{"index":7,"sense_key":"pay%2:31:00::","synset_id":"give.v.10"},{"index":8,"sense_key":"pay%2:42:00::","synset_id":"pay.v.07"},{"index":9,"sense_key":"pay%2:31:03::","synset_id":"pay.v.09"},{"index":10,"sense_key":"pay%2:40:01::","synset_id":"yield.v.10"},{"index":11,"sense_key":"pay%2:40:13::","synset_id":"pay.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"pay%2:40:14::","call_ids":["senseval3.d000.s122.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"pay%2:40:14::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":207,"reasoning_output_tokens":191},"cost":{"currency":"USD","total_usd":0.009705,"input_uncached_usd":0.0034950000000000003,"input_cached_usd":0.0,"output_usd":0.00621,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.558841541991569}
{"item_id":"senseval3.d000.s122.t001","gold_sense_keys":["bill%1:10:01::"],"candidates":[{"index":1,"sense_key":"bill%1:04:00::","synset_id":"bill.n.04"},{"index":2,"sense_key":"bill%1:10:02::","synset_id":"circular.n.01"},{"index":3,"sense_key":"bill%1:10:00::","synset_id":"poster.n.01"},{"index":4,"sense_key":"bill%1:10:04::","synset_id":"bill.n.01"},{"index":5,"sense_key":"bill%1:21:00::","synset_id":"bill.n.03"},{"index":6,"sense_key":"bill%1:06:00::","synset_id":"bill.n.09"},{"index":7,"sense_key":"bill%1:10:01::","synset_id":"bill.n.02"},{"index":8,"sense_key":"bill%1:05:00::","synset_id":"beak.n.02"},{"index":9,"sense_key":"bill%1:10:06::","synset_id":"bill.n.07"},{"index":10,"sense_key":"bill%1:06:01::","synset_id":"bill.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"bill%1:10:01::","call_ids":["senseval3.d000.s122.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"bill%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":733,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.005165,"input_uncached_usd":0.0036650000000000003,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.504015375045128}
{"item_id":"senseval3.d000.s123.t001","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d000.s123.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.003935,"input_uncached_usd":0.002255,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6356701659969985}
{"item_id":"senseval3.d000.s123.t002","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":2,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":5,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":6,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":7,"sense_key":"man%1:05:00::","synset_id":"world.n.08"},{"index":8,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":9,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":10,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":11,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:07::","call_ids":["senseval3.d000.s123.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:07::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":813,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019905,"input_uncached_usd":0.004065,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.85695020807907}
{"item_id":"senseval3.d000.s123.t004","gold_sense_keys":["say%2:32:01::","say%2:32:13::"],"candidates":[{"index":1,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":2,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":5,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":6,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":7,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":8,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":9,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":10,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":11,"sense_key":"say%2:32:04::","synset_id":"order.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval3.d000.s123.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":759,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019635,"input_uncached_usd":0.003795,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.419303457951173}
{"item_id":"senseval3.d000.s123.t005","gold_sense_keys":["divorce%2:41:00::"],"candidates":[{"index":1,"sense_key":"divorce%2:41:00::","synset_id":"divorce.v.02"},{"index":2,"sense_key":"divorce%2:41:01::","synset_id":"disassociate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"divorce%2:41:00::","call_ids":["senseval3.d000.s123.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"divorce%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":378,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.0039000000000000003,"input_uncached_usd":0.0018900000000000002,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6346317500574514}
{"item_id":"senseval3.d000.s125.t000","gold_sense_keys":["bitch%1:18:00::"],"candidates":[{"index":1,"sense_key":"bitch%1:18:00::","synset_id":"cunt.n.01"},{"index":2,"sense_key":"bitch%1:05:00::","synset_id":"bitch.n.04"},{"index":3,"sense_key":"bitch%1:26:00::","synset_id":"bitch.n.01"},{"index":4,"sense_key":"bitch%1:10:00::","synset_id":"gripe.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bitch%1:18:00::","call_ids":["senseval3.d000.s125.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bitch%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.0034200000000000003,"input_uncached_usd":0.0020700000000000002,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6015610420145094}
{"item_id":"senseval3.d000.s125.t001","gold_sense_keys":["love%2:37:02::"],"candidates":[{"index":1,"sense_key":"love%2:37:01::","synset_id":"love.v.03"},{"index":2,"sense_key":"love%2:35:00::","synset_id":"sleep_together.v.01"},{"index":3,"sense_key":"love%2:37:02::","synset_id":"love.v.02"},{"index":4,"sense_key":"love%2:37:00::","synset_id":"love.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"love%2:37:02::","call_ids":["senseval3.d000.s125.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"love%2:37:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":429,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.004665000000000001,"input_uncached_usd":0.002145,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.804805374937132}
{"item_id":"senseval3.d000.s126.t000","gold_sense_keys":["get%2:40:00::"],"candidates":[{"index":1,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":2,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":3,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":4,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":5,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":6,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":7,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":8,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":9,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":10,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":11,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":12,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":13,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":14,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":15,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":16,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":17,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":18,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":19,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":20,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":21,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":22,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":23,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":24,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":27,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":28,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":29,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"},{"index":30,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":31,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":32,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":33,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":34,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":35,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":36,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"get%2:40:00::","call_ids":["senseval3.d000.s126.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"get%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1860,"cached_input_tokens":0,"output_tokens":476,"reasoning_output_tokens":460},"cost":{"currency":"USD","total_usd":0.02358,"input_uncached_usd":0.009300000000000001,"input_cached_usd":0.0,"output_usd":0.014280000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.026245082961395}
{"item_id":"senseval3.d000.s126.t001","gold_sense_keys":["half%1:23:00::"],"candidates":[{"index":1,"sense_key":"half%1:28:00::","synset_id":"half.n.02"},{"index":2,"sense_key":"half%1:23:00::","synset_id":"one-half.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"half%1:23:00::","call_ids":["senseval3.d000.s126.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"half%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":306,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00333,"input_uncached_usd":0.0015300000000000001,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.801770874997601}
{"item_id":"senseval3.d000.s126.t002","gold_sense_keys":["have%2:40:00::","have%2:40:04::"],"candidates":[{"index":1,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":2,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":3,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":4,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":5,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":6,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":7,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":8,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":9,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":10,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":11,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":12,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":13,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":14,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":15,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":18,"sense_key":"have%2:35:00::","synset_id":"take.v.35"},{"index":19,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"have%2:40:04::","call_ids":["senseval3.d000.s126.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"have%2:40:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1062,"cached_input_tokens":0,"output_tokens":176,"reasoning_output_tokens":160},"cost":{"currency":"USD","total_usd":0.010590000000000002,"input_uncached_usd":0.0053100000000000005,"input_cached_usd":0.0,"output_usd":0.00528,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.910553833004087}
{"item_id":"senseval3.d000.s127.t001","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":3,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":4,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":5,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":6,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"},{"index":7,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":8,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":9,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"deal%1:10:00::","call_ids":["senseval3.d000.s127.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":4649,"reasoning_output_tokens":4583},"cost":{"currency":"USD","total_usd":0.142575,"input_uncached_usd":0.003105,"input_cached_usd":0.0,"output_usd":0.13947,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":93.54948091693223}
{"item_id":"senseval3.d000.s128.t001","gold_sense_keys":["business%1:14:00::"],"candidates":[{"index":1,"sense_key":"business%1:14:01::","synset_id":"clientele.n.01"},{"index":2,"sense_key":"business%1:14:00::","synset_id":"business.n.01"},{"index":3,"sense_key":"business%1:04:00::","synset_id":"occupation.n.01"},{"index":4,"sense_key":"business%1:04:02::","synset_id":"business.n.06"},{"index":5,"sense_key":"business%1:09:00::","synset_id":"business.n.05"},{"index":6,"sense_key":"business%1:14:02::","synset_id":"business.n.07"},{"index":7,"sense_key":"business%1:04:03::","synset_id":"business.n.09"},{"index":8,"sense_key":"business%1:09:01::","synset_id":"business.n.04"},{"index":9,"sense_key":"business%1:04:01::","synset_id":"commercial_enterprise.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"business%1:14:00::","call_ids":["senseval3.d000.s128.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"business%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":603,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.0055650000000000005,"input_uncached_usd":0.0030150000000000003,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.787710416945629}
{"item_id":"senseval3.d000.s128.t003","gold_sense_keys":["share%1:21:00::"],"candidates":[{"index":1,"sense_key":"share%1:04:00::","synset_id":"parcel.n.02"},{"index":2,"sense_key":"share%1:21:01::","synset_id":"share.n.02"},{"index":3,"sense_key":"share%1:21:00::","synset_id":"share.n.01"},{"index":4,"sense_key":"share%1:04:01::","synset_id":"contribution.n.01"},{"index":5,"sense_key":"share%1:06:00::","synset_id":"plowshare.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"share%1:21:00::","call_ids":["senseval3.d000.s128.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"share%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.004185,"input_uncached_usd":0.0023550000000000003,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4562315420480445}
{"item_id":"senseval3.d000.s130.t000","gold_sense_keys":["nod%2:32:00::"],"candidates":[{"index":1,"sense_key":"nod%2:29:03::","synset_id":"nod.v.03"},{"index":2,"sense_key":"nod%2:29:02::","synset_id":"nod.v.05"},{"index":3,"sense_key":"nod%2:32:01::","synset_id":"nod.v.01"},{"index":4,"sense_key":"nod%2:32:00::","synset_id":"nod.v.02"},{"index":5,"sense_key":"nod%2:38:00::","synset_id":"nod.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"nod%2:32:00::","call_ids":["senseval3.d000.s130.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"nod%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":393,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":88},"cost":{"currency":"USD","total_usd":0.005085,"input_uncached_usd":0.001965,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.292986667016521}
{"item_id":"senseval3.d000.s132.t000","gold_sense_keys":["see%2:31:01::"],"candidates":[{"index":1,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":2,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":3,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":4,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":5,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":6,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":7,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":8,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":9,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":10,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":11,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":12,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":13,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":14,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":15,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":16,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":17,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":18,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":19,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":20,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":21,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":22,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":23,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"see%2:31:01::","call_ids":["senseval3.d000.s132.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"see%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1345,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.008735000000000001,"input_uncached_usd":0.006725000000000001,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7994679589755833}
{"item_id":"senseval3.d000.s133.t000","gold_sense_keys":["understand%2:31:01::"],"candidates":[{"index":1,"sense_key":"understand%2:31:04::","synset_id":"sympathize.v.02"},{"index":2,"sense_key":"understand%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"understand%2:31:02::","synset_id":"understand.v.04"},{"index":4,"sense_key":"understand%2:31:03::","synset_id":"understand.v.03"},{"index":5,"sense_key":"understand%2:31:00::","synset_id":"understand.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"understand%2:31:01::","call_ids":["senseval3.d000.s133.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"understand%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":421,"cached_input_tokens":0,"output_tokens":1190,"reasoning_output_tokens":1163},"cost":{"currency":"USD","total_usd":0.037805000000000005,"input_uncached_usd":0.002105,"input_cached_usd":0.0,"output_usd":0.0357,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":21.335098791052587}
{"item_id":"senseval3.d000.s133.t001","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":4,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"},{"index":5,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":6,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":7,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":8,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"state%1:15:01::","call_ids":["senseval3.d000.s133.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"state%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.006030000000000001,"input_uncached_usd":0.0029700000000000004,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.148999375058338}
{"item_id":"senseval3.d000.s134.t000","gold_sense_keys":["come%2:38:00::","come%2:38:04::"],"candidates":[{"index":1,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":4,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":5,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":6,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":7,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":8,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":9,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":10,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":11,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":12,"sense_key":"come%2:31:13::","synset_id":"come.v.21"},{"index":13,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":14,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":15,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":16,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":17,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":18,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":19,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":20,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":21,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"come%2:38:04::","call_ids":["senseval3.d000.s134.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"come%2:38:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1074,"cached_input_tokens":0,"output_tokens":468,"reasoning_output_tokens":452},"cost":{"currency":"USD","total_usd":0.01941,"input_uncached_usd":0.005370000000000001,"input_cached_usd":0.0,"output_usd":0.01404,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.688794458052143}
{"item_id":"senseval3.d000.s134.t001","gold_sense_keys":["retire%2:41:01::"],"candidates":[{"index":1,"sense_key":"retire%2:35:00::","synset_id":"put_out.v.07"},{"index":2,"sense_key":"retire%2:41:04::","synset_id":"retire.v.04"},{"index":3,"sense_key":"retire%2:41:03::","synset_id":"retire.v.06"},{"index":4,"sense_key":"retire%2:40:00::","synset_id":"retire.v.07"},{"index":5,"sense_key":"retire%2:33:00::","synset_id":"retire.v.10"},{"index":6,"sense_key":"retire%2:41:02::","synset_id":"retire.v.02"},{"index":7,"sense_key":"retire%2:41:01::","synset_id":"retire.v.01"},{"index":8,"sense_key":"retire%2:38:00::","synset_id":"withdraw.v.01"},{"index":9,"sense_key":"retire%2:29:00::","synset_id":"go_to_bed.v.01"},{"index":10,"sense_key":"retire%2:37:00::","synset_id":"retire.v.08"},{"index":11,"sense_key":"retire%2:41:00::","synset_id":"adjourn.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"retire%2:41:01::","call_ids":["senseval3.d000.s134.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"retire%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":655,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.005405,"input_uncached_usd":0.003275,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.127483833115548}
{"item_id":"senseval3.d000.s135.t000","gold_sense_keys":["do%2:36:01::"],"candidates":[{"index":1,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":2,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":3,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":4,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":5,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":6,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":7,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":8,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":9,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":10,"sense_key":"do%2:38:00::","synset_id":"do.v.13"},{"index":11,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":12,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":13,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"do%2:41:04::","call_ids":["senseval3.d000.s135.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"do%2:41:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":801,"cached_input_tokens":0,"output_tokens":7001,"reasoning_output_tokens":6908},"cost":{"currency":"USD","total_usd":0.214035,"input_uncached_usd":0.004005000000000001,"input_cached_usd":0.0,"output_usd":0.21003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":99.56536516698543}
{"item_id":"senseval3.d000.s135.t001","gold_sense_keys":["free%5:00:00:unpaid:00"],"candidates":[{"index":1,"sense_key":"free%5:00:00:unfixed:00","synset_id":"detached.s.05"},{"index":2,"sense_key":"free%5:00:00:inexact:00","synset_id":"free.s.06"},{"index":3,"sense_key":"free%3:00:01::","synset_id":"free.a.06"},{"index":4,"sense_key":"free%3:00:02::","synset_id":"free.a.02"},{"index":5,"sense_key":"free%5:00:02:unoccupied:00","synset_id":"free.s.02"},{"index":6,"sense_key":"free%3:00:00::","synset_id":"free.a.01"},{"index":7,"sense_key":"free%5:00:00:unpaid:00","synset_id":"complimentary.s.01"},{"index":8,"sense_key":"free%5:00:00:nonexistent:00","synset_id":"barren.s.03"},{"index":9,"sense_key":"free%5:00:01:unoccupied:00","synset_id":"spare.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"free%5:00:00:unpaid:00","call_ids":["senseval3.d000.s135.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"free%5:00:00:unpaid:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":361,"reasoning_output_tokens":345},"cost":{"currency":"USD","total_usd":0.013815,"input_uncached_usd":0.0029850000000000002,"input_cached_usd":0.0,"output_usd":0.010830000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.690828458056785}
{"item_id":"senseval3.d000.s135.t002","gold_sense_keys":["job%1:04:01::"],"candidates":[{"index":1,"sense_key":"job%1:04:01::","synset_id":"job.n.02"},{"index":2,"sense_key":"job%1:18:01::","synset_id":"job.n.10"},{"index":3,"sense_key":"job%1:04:03::","synset_id":"job.n.07"},{"index":4,"sense_key":"job%1:04:05::","synset_id":"job.n.06"},{"index":5,"sense_key":"job%1:04:00::","synset_id":"occupation.n.01"},{"index":6,"sense_key":"job%1:04:04::","synset_id":"job.n.05"},{"index":7,"sense_key":"job%1:10:00::","synset_id":"job.n.12"},{"index":8,"sense_key":"job%1:10:01::","synset_id":"job.n.11"},{"index":9,"sense_key":"job%1:18:00::","synset_id":"job.n.09"},{"index":10,"sense_key":"job%1:04:02::","synset_id":"caper.n.03"},{"index":11,"sense_key":"job%1:06:01::","synset_id":"job.n.04"},{"index":12,"sense_key":"job%1:26:00::","synset_id":"problem.n.01"},{"index":13,"sense_key":"job%1:06:00::","synset_id":"job.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"job%1:04:01::","call_ids":["senseval3.d000.s135.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"job%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":296,"reasoning_output_tokens":281},"cost":{"currency":"USD","total_usd":0.012645,"input_uncached_usd":0.003765,"input_cached_usd":0.0,"output_usd":0.00888,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.141819374985062}
{"item_id":"senseval3.d000.s136.t000","gold_sense_keys":["tell%2:32:00::","tell%2:32:04::"],"candidates":[{"index":1,"sense_key":"tell%2:31:00::","synset_id":"tell.v.05"},{"index":2,"sense_key":"tell%2:32:03::","synset_id":"tell.v.07"},{"index":3,"sense_key":"tell%2:32:02::","synset_id":"order.v.01"},{"index":4,"sense_key":"tell%2:32:04::","synset_id":"state.v.01"},{"index":5,"sense_key":"tell%2:31:01::","synset_id":"distinguish.v.01"},{"index":6,"sense_key":"tell%2:32:01::","synset_id":"tell.v.03"},{"index":7,"sense_key":"tell%2:32:08::","synset_id":"assure.v.02"},{"index":8,"sense_key":"tell%2:32:00::","synset_id":"tell.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"tell%2:32:00::","call_ids":["senseval3.d000.s136.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"tell%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":110},"cost":{"currency":"USD","total_usd":0.006535,"input_uncached_usd":0.002755,"input_cached_usd":0.0,"output_usd":0.00378,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.239837583969347}
{"item_id":"senseval3.d000.s137.t000","gold_sense_keys":["feel%2:29:00::","feel%2:37:00::"],"candidates":[{"index":1,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":2,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":3,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":6,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"},{"index":9,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":10,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":11,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":12,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":13,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"feel%2:37:00::","call_ids":["senseval3.d000.s137.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"feel%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":101},"cost":{"currency":"USD","total_usd":0.00736,"input_uncached_usd":0.00385,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.132442708942108}
{"item_id":"senseval3.d000.s137.t001","gold_sense_keys":["twinge%1:12:00::"],"candidates":[{"index":1,"sense_key":"twinge%1:12:00::","synset_id":"pang.n.01"},{"index":2,"sense_key":"twinge%1:09:00::","synset_id":"twinge.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"twinge%1:12:00::","call_ids":["senseval3.d000.s137.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"twinge%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":291,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.0033150000000000002,"input_uncached_usd":0.0014550000000000001,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7223361249780282}
{"item_id":"senseval3.d000.s137.t002","gold_sense_keys":["annoyance%1:12:00::"],"candidates":[{"index":1,"sense_key":"annoyance%1:18:00::","synset_id":"aggravator.n.01"},{"index":2,"sense_key":"annoyance%1:12:00::","synset_id":"annoyance.n.02"},{"index":3,"sense_key":"annoyance%1:26:00::","synset_id":"irritation.n.01"},{"index":4,"sense_key":"annoyance%1:09:00::","synset_id":"annoyance.n.04"},{"index":5,"sense_key":"annoyance%1:04:00::","synset_id":"annoyance.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"annoyance%1:12:00::","call_ids":["senseval3.d000.s137.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"annoyance%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":628,"reasoning_output_tokens":606},"cost":{"currency":"USD","total_usd":0.021060000000000002,"input_uncached_usd":0.00222,"input_cached_usd":0.0,"output_usd":0.01884,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.627059750026092}
{"item_id":"senseval3.d000.s137.t003","gold_sense_keys":["hear%2:39:00::"],"candidates":[{"index":1,"sense_key":"hear%2:39:05::","synset_id":"listen.v.02"},{"index":2,"sense_key":"hear%2:31:00::","synset_id":"learn.v.02"},{"index":3,"sense_key":"hear%2:39:01::","synset_id":"hear.v.04"},{"index":4,"sense_key":"hear%2:39:00::","synset_id":"hear.v.01"},{"index":5,"sense_key":"hear%2:41:00::","synset_id":"hear.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"hear%2:39:00::","call_ids":["senseval3.d000.s137.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"hear%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":117},"cost":{"currency":"USD","total_usd":0.0062450000000000006,"input_uncached_usd":0.002255,"input_cached_usd":0.0,"output_usd":0.0039900000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.015230958000757}
{"item_id":"senseval3.d000.s137.t004","gold_sense_keys":["familiar%3:00:00::"],"candidates":[{"index":1,"sense_key":"familiar%5:00:00:close:02","synset_id":"familiar.s.02"},{"index":2,"sense_key":"familiar%3:00:02::","synset_id":"familiar.a.02"},{"index":3,"sense_key":"familiar%3:00:00::","synset_id":"familiar.a.01"},{"index":4,"sense_key":"familiar%5:00:00:informed:00","synset_id":"conversant.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"familiar%3:00:00::","call_ids":["senseval3.d000.s137.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"familiar%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":406,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.00371,"input_uncached_usd":0.00203,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9798546250676736}
{"item_id":"senseval3.d000.s138.t000","gold_sense_keys":["wild%5:00:00:unrealistic:00"],"candidates":[{"index":1,"sense_key":"wild%5:00:00:unrealistic:00","synset_id":"fantastic.s.03"},{"index":2,"sense_key":"wild%5:00:00:passionate:00","synset_id":"wild.s.01"},{"index":3,"sense_key":"wild%5:00:00:intense:00","synset_id":"violent.s.02"},{"index":4,"sense_key":"wild%3:00:01::","synset_id":"wild.a.02"},{"index":5,"sense_key":"wild%5:00:00:noncivilized:00","synset_id":"barbarian.s.01"},{"index":6,"sense_key":"wild%5:00:00:unsupported:02","synset_id":"baseless.s.01"},{"index":7,"sense_key":"wild%5:00:00:stormy:00","synset_id":"angry.s.01"},{"index":8,"sense_key":"wild%5:00:00:insane:00","synset_id":"raving_mad.s.01"},{"index":9,"sense_key":"wild%5:00:01:uncontrolled:00","synset_id":"wild.s.02"},{"index":10,"sense_key":"wild%5:00:00:enthusiastic:00","synset_id":"crazy.s.05"},{"index":11,"sense_key":"wild%3:00:02::","synset_id":"wild.a.01"},{"index":12,"sense_key":"wild%5:00:00:inhospitable:00","synset_id":"godforsaken.s.01"},{"index":13,"sense_key":"wild%5:00:00:dangerous:00","synset_id":"hazardous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wild%5:00:00:unrealistic:00","call_ids":["senseval3.d000.s138.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wild%5:00:00:unrealistic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":854,"cached_input_tokens":0,"output_tokens":297,"reasoning_output_tokens":282},"cost":{"currency":"USD","total_usd":0.01318,"input_uncached_usd":0.00427,"input_cached_usd":0.0,"output_usd":0.00891,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.200298000010662}
{"item_id":"senseval3.d000.s138.t001","gold_sense_keys":["thought%1:09:01::"],"candidates":[{"index":1,"sense_key":"thought%1:09:02::","synset_id":"thought.n.03"},{"index":2,"sense_key":"thought%1:09:03::","synset_id":"opinion.n.01"},{"index":3,"sense_key":"thought%1:09:01::","synset_id":"idea.n.01"},{"index":4,"sense_key":"thought%1:09:00::","synset_id":"thinking.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"thought%1:09:01::","call_ids":["senseval3.d000.s138.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"thought%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.004305,"input_uncached_usd":0.0021750000000000003,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.259360041934997}
{"item_id":"senseval3.d000.s138.t004","gold_sense_keys":["clouded%5:00:00:confused:00"],"candidates":[{"index":1,"sense_key":"clouded%5:00:00:troubled:00","synset_id":"clouded.s.01"},{"index":2,"sense_key":"clouded%5:00:00:unclear:00","synset_id":"blurred.s.02"},{"index":3,"sense_key":"clouded%5:00:00:confused:00","synset_id":"clouded.s.03"},{"index":4,"sense_key":"clouded%5:00:00:cloudy:00","synset_id":"cloud-covered.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"clouded%5:00:00:confused:00","call_ids":["senseval3.d000.s138.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"clouded%5:00:00:confused:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":124},"cost":{"currency":"USD","total_usd":0.006275000000000001,"input_uncached_usd":0.002075,"input_cached_usd":0.0,"output_usd":0.0042,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.514877457986586}
{"item_id":"senseval3.d000.s138.t005","gold_sense_keys":["brain%1:09:01::"],"candidates":[{"index":1,"sense_key":"brain%1:09:01::","synset_id":"mind.n.01"},{"index":2,"sense_key":"brain%1:08:00::","synset_id":"brain.n.01"},{"index":3,"sense_key":"brain%1:18:00::","synset_id":"genius.n.01"},{"index":4,"sense_key":"brain%1:13:00::","synset_id":"brain.n.05"},{"index":5,"sense_key":"brain%1:09:00::","synset_id":"brain.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brain%1:09:01::","call_ids":["senseval3.d000.s138.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brain%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":468,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.00546,"input_uncached_usd":0.00234,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6144008330302313}
{"item_id":"senseval3.d000.s139.t000","gold_sense_keys":["suppose%2:32:00::"],"candidates":[{"index":1,"sense_key":"suppose%2:31:05::","synset_id":"presuppose.v.02"},{"index":2,"sense_key":"suppose%2:32:00::","synset_id":"suppose.v.01"},{"index":3,"sense_key":"suppose%2:31:06::","synset_id":"presuppose.v.01"},{"index":4,"sense_key":"suppose%2:31:00::","synset_id":"think.v.02"},{"index":5,"sense_key":"suppose%2:31:01::","synset_id":"speculate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"suppose%2:32:00::","call_ids":["senseval3.d000.s139.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"suppose%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":245,"reasoning_output_tokens":229},"cost":{"currency":"USD","total_usd":0.009815,"input_uncached_usd":0.002465,"input_cached_usd":0.0,"output_usd":0.00735,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.088652417063713}
{"item_id":"senseval3.d000.s139.t001","gold_sense_keys":["suppose%2:32:00::"],"candidates":[{"index":1,"sense_key":"suppose%2:31:06::","synset_id":"presuppose.v.01"},{"index":2,"sense_key":"suppose%2:31:00::","synset_id":"think.v.02"},{"index":3,"sense_key":"suppose%2:31:05::","synset_id":"presuppose.v.02"},{"index":4,"sense_key":"suppose%2:31:01::","synset_id":"speculate.v.01"},{"index":5,"sense_key":"suppose%2:32:00::","synset_id":"suppose.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"suppose%2:32:00::","call_ids":["senseval3.d000.s139.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"suppose%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":350,"reasoning_output_tokens":334},"cost":{"currency":"USD","total_usd":0.012964999999999999,"input_uncached_usd":0.002465,"input_cached_usd":0.0,"output_usd":0.0105,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.74357395793777}
{"item_id":"senseval3.d000.s139.t002","gold_sense_keys":["guy%1:18:00::"],"candidates":[{"index":1,"sense_key":"guy%1:18:00::","synset_id":"guy.n.01"},{"index":2,"sense_key":"guy%1:06:00::","synset_id":"guy.n.03"},{"index":3,"sense_key":"guy%1:06:01::","synset_id":"guy.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"guy%1:18:00::","call_ids":["senseval3.d000.s139.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"guy%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":361,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.003125,"input_uncached_usd":0.0018050000000000002,"input_cached_usd":0.0,"output_usd":0.00132,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.881750209024176}
{"item_id":"senseval3.d000.s139.t003","gold_sense_keys":["say%2:32:01::"],"candidates":[{"index":1,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":2,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":3,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":4,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":5,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":6,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":9,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":10,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":11,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval3.d000.s139.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":1909,"reasoning_output_tokens":1875},"cost":{"currency":"USD","total_usd":0.060825000000000004,"input_uncached_usd":0.0035550000000000004,"input_cached_usd":0.0,"output_usd":0.05727,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":37.708421332994476}
{"item_id":"senseval3.d000.s140.t001","gold_sense_keys":["professional%3:00:01::"],"candidates":[{"index":1,"sense_key":"professional%3:01:00::","synset_id":"professional.a.04"},{"index":2,"sense_key":"professional%3:00:01::","synset_id":"professional.a.01"},{"index":3,"sense_key":"professional%5:00:00:white-collar:00","synset_id":"professional.s.01"},{"index":4,"sense_key":"professional%3:01:01::","synset_id":"professional.a.02"},{"index":5,"sense_key":"professional%3:00:02::","synset_id":"professional.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"professional%3:00:01::","call_ids":["senseval3.d000.s140.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"professional%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":113},"cost":{"currency":"USD","total_usd":0.00601,"input_uncached_usd":0.00217,"input_cached_usd":0.0,"output_usd":0.00384,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.252405792009085}
{"item_id":"senseval3.d000.s140.t002","gold_sense_keys":["killer%1:18:00::"],"candidates":[{"index":1,"sense_key":"killer%1:18:00::","synset_id":"killer.n.01"},{"index":2,"sense_key":"killer%1:05:00::","synset_id":"killer_whale.n.01"},{"index":3,"sense_key":"killer%1:26:00::","synset_id":"cause_of_death.n.01"},{"index":4,"sense_key":"killer%1:09:00::","synset_id":"killer.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"killer%1:18:00::","call_ids":["senseval3.d000.s140.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"killer%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":392,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.0034300000000000003,"input_uncached_usd":0.0019600000000000004,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3948123339796439}
{"item_id":"senseval3.d000.s141.t000","gold_sense_keys":["nut%1:18:02::"],"candidates":[{"index":1,"sense_key":"nut%1:18:02::","synset_id":"crackpot.n.01"},{"index":2,"sense_key":"nut%1:23:00::","synset_id":"en.n.01"},{"index":3,"sense_key":"nut%1:18:01::","synset_id":"addict.n.01"},{"index":4,"sense_key":"nut%1:08:00::","synset_id":"testis.n.01"},{"index":5,"sense_key":"nut%1:06:00::","synset_id":"nut.n.03"},{"index":6,"sense_key":"nut%1:20:00::","synset_id":"nut.n.01"},{"index":7,"sense_key":"nut%1:18:00::","synset_id":"nut.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nut%1:18:02::","call_ids":["senseval3.d000.s141.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nut%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.00469,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.679323707940057}
{"item_id":"senseval3.d000.s141.t001","gold_sense_keys":["harm%1:11:01::"],"candidates":[{"index":1,"sense_key":"harm%1:11:01::","synset_id":"damage.n.01"},{"index":2,"sense_key":"harm%1:26:00::","synset_id":"injury.n.01"},{"index":3,"sense_key":"harm%1:04:00::","synset_id":"damage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"harm%1:11:01::","call_ids":["senseval3.d000.s141.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"harm%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":339,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017535,"input_uncached_usd":0.0016950000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.646848416072316}
{"item_id":"senseval3.d000.s141.t002","gold_sense_keys":["do%2:36:02::"],"candidates":[{"index":1,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":2,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":3,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":4,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":5,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":6,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":7,"sense_key":"do%2:38:00::","synset_id":"do.v.13"},{"index":8,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":9,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":10,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":11,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":12,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":13,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"do%2:36:02::","call_ids":["senseval3.d000.s141.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"do%2:36:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":830,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":109},"cost":{"currency":"USD","total_usd":0.00787,"input_uncached_usd":0.00415,"input_cached_usd":0.0,"output_usd":0.00372,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5865166659932584}
{"item_id":"senseval3.d000.s142.t001","gold_sense_keys":["do%2:36:01::","do%2:41:01::","do%2:41:04::"],"candidates":[{"index":1,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":2,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":3,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":4,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":5,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":6,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":7,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":8,"sense_key":"do%2:38:00::","synset_id":"do.v.13"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":11,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":12,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":13,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"do%2:36:01::","call_ids":["senseval3.d000.s142.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":841,"cached_input_tokens":0,"output_tokens":164,"reasoning_output_tokens":149},"cost":{"currency":"USD","total_usd":0.009125000000000001,"input_uncached_usd":0.004205,"input_cached_usd":0.0,"output_usd":0.00492,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.561075208010152}
{"item_id":"senseval3.d000.s143.t000","gold_sense_keys":["feel%2:29:00::","feel%2:37:00::"],"candidates":[{"index":1,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":2,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":3,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":4,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":5,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":6,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":7,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":8,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":9,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":10,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"},{"index":11,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":12,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":13,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"feel%2:31:01::","call_ids":["senseval3.d000.s143.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"feel%2:31:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":780,"cached_input_tokens":0,"output_tokens":2589,"reasoning_output_tokens":2549},"cost":{"currency":"USD","total_usd":0.08157,"input_uncached_usd":0.0039000000000000003,"input_cached_usd":0.0,"output_usd":0.07767,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":47.266184792038985}
{"item_id":"senseval3.d000.s143.t001","gold_sense_keys":["cunning%5:00:00:adroit:00","cunning%5:00:00:artful:00"],"candidates":[{"index":1,"sense_key":"cunning%5:00:00:attractive:01","synset_id":"cunning.s.01"},{"index":2,"sense_key":"cunning%5:00:00:artful:00","synset_id":"crafty.s.01"},{"index":3,"sense_key":"cunning%5:00:00:adroit:00","synset_id":"clever.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cunning%5:00:00:artful:00","call_ids":["senseval3.d000.s143.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cunning%5:00:00:artful:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":398,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":110},"cost":{"currency":"USD","total_usd":0.00577,"input_uncached_usd":0.00199,"input_cached_usd":0.0,"output_usd":0.00378,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5922345409635454}
{"item_id":"senseval3.d000.s143.t002","gold_sense_keys":["proud%3:00:00::"],"candidates":[{"index":1,"sense_key":"proud%3:00:00::","synset_id":"proud.a.01"},{"index":2,"sense_key":"proud%5:00:00:impressive:00","synset_id":"gallant.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"proud%3:00:00::","call_ids":["senseval3.d000.s143.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"proud%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":337,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.0036049999999999997,"input_uncached_usd":0.001685,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7289660420501605}
{"item_id":"senseval3.d000.s143.t003","gold_sense_keys":["play%2:34:13::"],"candidates":[{"index":1,"sense_key":"play%2:38:03::","synset_id":"play.v.24"},{"index":2,"sense_key":"play%2:41:13::","synset_id":"dally.v.01"},{"index":3,"sense_key":"play%2:33:01::","synset_id":"play.v.10"},{"index":4,"sense_key":"play%2:33:08::","synset_id":"bet.v.02"},{"index":5,"sense_key":"play%2:36:06::","synset_id":"play.v.12"},{"index":6,"sense_key":"play%2:33:12::","synset_id":"play.v.33"},{"index":7,"sense_key":"play%2:36:13::","synset_id":"bring.v.03"},{"index":8,"sense_key":"play%2:38:00::","synset_id":"play.v.09"},{"index":9,"sense_key":"play%2:29:04::","synset_id":"play.v.35"},{"index":10,"sense_key":"play%2:41:03::","synset_id":"play.v.05"},{"index":11,"sense_key":"play%2:36:01::","synset_id":"play.v.07"},{"index":12,"sense_key":"play%2:41:00::","synset_id":"play.v.11"},{"index":13,"sense_key":"play%2:36:12::","synset_id":"play.v.03"},{"index":14,"sense_key":"play%2:36:00::","synset_id":"play.v.06"},{"index":15,"sense_key":"play%2:36:04::","synset_id":"play.v.26"},{"index":16,"sense_key":"play%2:42:00::","synset_id":"play.v.22"},{"index":17,"sense_key":"play%2:36:11::","synset_id":"play.v.13"},{"index":18,"sense_key":"play%2:33:00::","synset_id":"play.v.01"},{"index":19,"sense_key":"play%2:35:00::","synset_id":"toy.v.02"},{"index":20,"sense_key":"play%2:36:05::","synset_id":"play.v.18"},{"index":21,"sense_key":"play%2:33:09::","synset_id":"play.v.31"},{"index":22,"sense_key":"play%2:33:02::","synset_id":"meet.v.10"},{"index":23,"sense_key":"play%2:34:13::","synset_id":"play.v.20"},{"index":24,"sense_key":"play%2:31:01::","synset_id":"dally.v.04"},{"index":25,"sense_key":"play%2:36:02::","synset_id":"act.v.03"},{"index":26,"sense_key":"play%2:33:13::","synset_id":"play.v.32"},{"index":27,"sense_key":"play%2:29:00::","synset_id":"play.v.16"},{"index":28,"sense_key":"play%2:41:12::","synset_id":"play.v.02"},{"index":29,"sense_key":"play%2:41:02::","synset_id":"play.v.17"},{"index":30,"sense_key":"play%2:35:07::","synset_id":"play.v.28"},{"index":31,"sense_key":"play%2:29:01::","synset_id":"act.v.05"},{"index":32,"sense_key":"play%2:36:10::","synset_id":"act.v.10"},{"index":33,"sense_key":"play%2:36:03::","synset_id":"play.v.14"},{"index":34,"sense_key":"play%2:33:03::","synset_id":"play.v.15"},{"index":35,"sense_key":"play%2:33:07::","synset_id":"play.v.29"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":23,"chosen_sense_key":"play%2:34:13::","call_ids":["senseval3.d000.s143.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":23,"predicted_sense_key":"play%2:34:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1772,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.011980000000000001,"input_uncached_usd":0.008860000000000002,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3066790000302717}
{"item_id":"senseval3.d000.s143.t004","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":2,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%3:00:00::","synset_id":"other.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d000.s143.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.0040550000000000004,"input_uncached_usd":0.0020450000000000004,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.603744167019613}
{"item_id":"senseval3.d000.s143.t005","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":2,"sense_key":"man%1:05:00::","synset_id":"world.n.08"},{"index":3,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":4,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":5,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":6,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":7,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":8,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":11,"sense_key":"man%1:18:07::","synset_id":"man.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s143.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":73},"cost":{"currency":"USD","total_usd":0.006505,"input_uncached_usd":0.0038650000000000004,"input_cached_usd":0.0,"output_usd":0.00264,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5676610830705613}
{"item_id":"senseval3.d000.s143.t006","gold_sense_keys":["soft_spot%1:07:00::"],"candidates":[{"index":1,"sense_key":"soft_spot%1:08:00::","synset_id":"fontanelle.n.01"},{"index":2,"sense_key":"soft_spot%1:12:00::","synset_id":"soft_spot.n.02"},{"index":3,"sense_key":"soft_spot%1:07:00::","synset_id":"weak_part.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"soft_spot%1:07:00::","call_ids":["senseval3.d000.s143.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"soft_spot%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":346,"cached_input_tokens":0,"output_tokens":288,"reasoning_output_tokens":272},"cost":{"currency":"USD","total_usd":0.01037,"input_uncached_usd":0.0017300000000000002,"input_cached_usd":0.0,"output_usd":0.00864,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.778053332935087}
{"item_id":"senseval3.d000.s144.t000","gold_sense_keys":["offense%1:04:02::"],"candidates":[{"index":1,"sense_key":"offense%1:04:01::","synset_id":"crime.n.01"},{"index":2,"sense_key":"offense%1:04:00::","synset_id":"offense.n.05"},{"index":3,"sense_key":"offense%1:12:00::","synset_id":"umbrage.n.01"},{"index":4,"sense_key":"offense%1:04:02::","synset_id":"discourtesy.n.03"},{"index":5,"sense_key":"offense%1:14:00::","synset_id":"offense.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"offense%1:04:02::","call_ids":["senseval3.d000.s144.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"offense%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":294,"reasoning_output_tokens":278},"cost":{"currency":"USD","total_usd":0.011065,"input_uncached_usd":0.002245,"input_cached_usd":0.0,"output_usd":0.00882,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.7583319580880925}
{"item_id":"senseval3.d000.s144.t001","gold_sense_keys":["intend%2:31:00::"],"candidates":[{"index":1,"sense_key":"intend%2:31:01::","synset_id":"intend.v.02"},{"index":2,"sense_key":"intend%2:32:02::","synset_id":"mean.v.03"},{"index":3,"sense_key":"intend%2:31:00::","synset_id":"intend.v.01"},{"index":4,"sense_key":"intend%2:32:00::","synset_id":"mean.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"intend%2:31:00::","call_ids":["senseval3.d000.s144.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"intend%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.0048400000000000006,"input_uncached_usd":0.0020800000000000003,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8103524170583114}
{"item_id":"senseval3.d000.s144.t002","gold_sense_keys":["say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":2,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":3,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":4,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":5,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":6,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":7,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":8,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":9,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":10,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":11,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"say%2:32:15::","call_ids":["senseval3.d000.s144.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"say%2:32:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":718,"cached_input_tokens":0,"output_tokens":330,"reasoning_output_tokens":314},"cost":{"currency":"USD","total_usd":0.013489999999999999,"input_uncached_usd":0.0035900000000000003,"input_cached_usd":0.0,"output_usd":0.0099,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.275072708027437}
{"item_id":"senseval3.d000.s146.t001","gold_sense_keys":["suspect%2:32:00::"],"candidates":[{"index":1,"sense_key":"suspect%2:32:02::","synset_id":"suspect.v.03"},{"index":2,"sense_key":"suspect%2:31:00::","synset_id":"distrust.v.01"},{"index":3,"sense_key":"suspect%2:32:00::","synset_id":"suspect.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"suspect%2:32:00::","call_ids":["senseval3.d000.s146.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"suspect%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":381,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":94},"cost":{"currency":"USD","total_usd":0.005205,"input_uncached_usd":0.0019050000000000002,"input_cached_usd":0.0,"output_usd":0.0033,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.462158624897711}
{"item_id":"senseval3.d000.s146.t002","gold_sense_keys":["thing%1:06:01::"],"candidates":[{"index":1,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":2,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":3,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":4,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":5,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":6,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":7,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":8,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":9,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":10,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":11,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"thing%1:06:01::","call_ids":["senseval3.d000.s146.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"thing%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":9000,"reasoning_output_tokens":8879},"cost":{"currency":"USD","total_usd":0.27318000000000003,"input_uncached_usd":0.00318,"input_cached_usd":0.0,"output_usd":0.27,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":141.00113550003152}
{"item_id":"senseval3.d000.s146.t003","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":2,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":3,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":4,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":5,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":6,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:30:00::","synset_id":"think.v.13"},{"index":9,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":12,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":13,"sense_key":"think%2:31:09::","synset_id":"think.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval3.d000.s146.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":808,"cached_input_tokens":0,"output_tokens":588,"reasoning_output_tokens":566},"cost":{"currency":"USD","total_usd":0.021680000000000005,"input_uncached_usd":0.00404,"input_cached_usd":0.0,"output_usd":0.01764,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.706101416959427}
{"item_id":"senseval3.d000.s146.t004","gold_sense_keys":["coincidence%1:11:00::"],"candidates":[{"index":1,"sense_key":"coincidence%1:07:02::","synset_id":"coincidence.n.02"},{"index":2,"sense_key":"coincidence%1:11:00::","synset_id":"coincidence.n.01"},{"index":3,"sense_key":"coincidence%1:07:01::","synset_id":"concurrence.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"coincidence%1:11:00::","call_ids":["senseval3.d000.s146.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"coincidence%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":375,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.004125,"input_uncached_usd":0.0018750000000000001,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.091050749993883}
{"item_id":"senseval3.d000.s147.t000","gold_sense_keys":["know%2:31:00::"],"candidates":[{"index":1,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":2,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":3,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":4,"sense_key":"know%2:31:14::","synset_id":"know.v.11"},{"index":5,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":6,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":7,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":10,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":11,"sense_key":"know%2:31:02::","synset_id":"know.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"know%2:31:00::","call_ids":["senseval3.d000.s147.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"know%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":750,"cached_input_tokens":0,"output_tokens":1094,"reasoning_output_tokens":1066},"cost":{"currency":"USD","total_usd":0.036570000000000005,"input_uncached_usd":0.0037500000000000003,"input_cached_usd":0.0,"output_usd":0.03282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":17.689234958030283}
{"item_id":"senseval3.d000.s150.t000","gold_sense_keys":["say%2:32:01::"],"candidates":[{"index":1,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":2,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":5,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":6,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":7,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":8,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":9,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":10,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":11,"sense_key":"say%2:32:08::","synset_id":"say.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval3.d000.s150.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":1564,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.050375,"input_uncached_usd":0.003455,"input_cached_usd":0.0,"output_usd":0.04692,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":31.12956795899663}
{"item_id":"senseval3.d000.s150.t001","gold_sense_keys":["manage%2:41:09::"],"candidates":[{"index":1,"sense_key":"manage%2:41:02::","synset_id":"do.v.11"},{"index":2,"sense_key":"manage%2:35:03::","synset_id":"wield.v.02"},{"index":3,"sense_key":"manage%2:41:08::","synset_id":"wangle.v.01"},{"index":4,"sense_key":"manage%2:41:03::","synset_id":"cope.v.01"},{"index":5,"sense_key":"manage%2:41:01::","synset_id":"oversee.v.01"},{"index":6,"sense_key":"manage%2:41:00::","synset_id":"manage.v.02"},{"index":7,"sense_key":"manage%2:41:09::","synset_id":"pull_off.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"manage%2:41:08::","call_ids":["senseval3.d000.s150.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"manage%2:41:08::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018515,"input_uncached_usd":0.0026750000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.570182582945563}
{"item_id":"senseval3.d000.s151.t000","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"},{"index":2,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s151.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":267,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":116},"cost":{"currency":"USD","total_usd":0.005265000000000001,"input_uncached_usd":0.001335,"input_cached_usd":0.0,"output_usd":0.00393,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8364679170772433}
{"item_id":"senseval3.d000.s151.t001","gold_sense_keys":["hook%2:32:09::"],"candidates":[{"index":1,"sense_key":"hook%2:33:00::","synset_id":"hook.v.10"},{"index":2,"sense_key":"hook%2:35:01::","synset_id":"hook.v.08"},{"index":3,"sense_key":"hook%2:32:09::","synset_id":"hook.v.11"},{"index":4,"sense_key":"hook%2:35:00::","synset_id":"hook.v.01"},{"index":5,"sense_key":"hook%2:35:02::","synset_id":"hook.v.07"},{"index":6,"sense_key":"hook%2:40:01::","synset_id":"hook.v.05"},{"index":7,"sense_key":"hook%2:40:02::","synset_id":"overcharge.v.01"},{"index":8,"sense_key":"hook%2:32:00::","synset_id":"hook.v.12"},{"index":9,"sense_key":"hook%2:35:03::","synset_id":"hook.v.04"},{"index":10,"sense_key":"hook%2:36:00::","synset_id":"crochet.v.02"},{"index":11,"sense_key":"hook%2:40:00::","synset_id":"pilfer.v.01"},{"index":12,"sense_key":"hook%2:34:00::","synset_id":"addict.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"hook%2:32:09::","call_ids":["senseval3.d000.s151.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"hook%2:32:09::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":116},"cost":{"currency":"USD","total_usd":0.0072,"input_uncached_usd":0.0032400000000000003,"input_cached_usd":0.0,"output_usd":0.00396,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.835583750042133}
{"item_id":"senseval3.d000.s152.t000","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":2,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":3,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"},{"index":4,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":5,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s152.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":397,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.003515,"input_uncached_usd":0.0019850000000000002,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1761313340393826}
{"item_id":"senseval3.d000.s152.t001","gold_sense_keys":["burn%2:43:01::"],"candidates":[{"index":1,"sense_key":"burn%2:37:00::","synset_id":"burn.v.06"},{"index":2,"sense_key":"burn%2:39:04::","synset_id":"burn.v.10"},{"index":3,"sense_key":"burn%2:36:05::","synset_id":"cut.v.21"},{"index":4,"sense_key":"burn%2:40:01::","synset_id":"burn.v.09"},{"index":5,"sense_key":"burn%2:30:08::","synset_id":"cauterize.v.01"},{"index":6,"sense_key":"burn%2:30:00::","synset_id":"burn.v.03"},{"index":7,"sense_key":"burn%2:30:01::","synset_id":"burn.v.07"},{"index":8,"sense_key":"burn%2:34:00::","synset_id":"burn_off.v.01"},{"index":9,"sense_key":"burn%2:43:02::","synset_id":"burn.v.05"},{"index":10,"sense_key":"burn%2:30:03::","synset_id":"burn.v.15"},{"index":11,"sense_key":"burn%2:29:00::","synset_id":"sunburn.v.01"},{"index":12,"sense_key":"burn%2:39:00::","synset_id":"bite.v.02"},{"index":13,"sense_key":"burn%2:30:02::","synset_id":"burn.v.01"},{"index":14,"sense_key":"burn%2:41:00::","synset_id":"burn.v.08"},{"index":15,"sense_key":"burn%2:43:01::","synset_id":"burn.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"burn%2:43:01::","call_ids":["senseval3.d000.s152.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"burn%2:43:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01971,"input_uncached_usd":0.00387,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":510.67511683295015}
{"item_id":"senseval3.d000.s153.t000","gold_sense_keys":["mutter%2:32:01::"],"candidates":[{"index":1,"sense_key":"mutter%2:32:01::","synset_id":"mumble.v.01"},{"index":2,"sense_key":"mutter%2:32:00::","synset_id":"murmur.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mutter%2:32:01::","call_ids":["senseval3.d000.s153.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mutter%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":289,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":117},"cost":{"currency":"USD","total_usd":0.005435000000000001,"input_uncached_usd":0.001445,"input_cached_usd":0.0,"output_usd":0.0039900000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.01157020800747}
{"item_id":"senseval3.d000.s154.t001","gold_sense_keys":["look%2:39:01::"],"candidates":[{"index":1,"sense_key":"look%2:31:02::","synset_id":"count.v.08"},{"index":2,"sense_key":"look%2:29:00::","synset_id":"look.v.03"},{"index":3,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":4,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":5,"sense_key":"look%2:39:00::","synset_id":"look.v.01"},{"index":6,"sense_key":"look%2:42:00::","synset_id":"front.v.01"},{"index":7,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":8,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":9,"sense_key":"look%2:39:01::","synset_id":"look.v.02"},{"index":10,"sense_key":"look%2:39:02::","synset_id":"search.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"look%2:39:01::","call_ids":["senseval3.d000.s154.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"look%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":656,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.00523,"input_uncached_usd":0.0032800000000000004,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.133139916928485}
{"item_id":"senseval3.d000.s155.t001","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":2,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":3,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":4,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":5,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":6,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":7,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":8,"sense_key":"way%1:21:00::","synset_id":"way.n.12"},{"index":9,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":12,"sense_key":"way%1:06:00::","synset_id":"way.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"way%1:07:01::","call_ids":["senseval3.d000.s155.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.005945000000000001,"input_uncached_usd":0.003425,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0730253749061376}
{"item_id":"senseval3.d000.s155.t002","gold_sense_keys":["time%1:28:06::"],"candidates":[{"index":1,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":2,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":3,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":4,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":5,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":6,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":9,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"time%1:28:06::","call_ids":["senseval3.d000.s155.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"time%1:28:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":398,"reasoning_output_tokens":382},"cost":{"currency":"USD","total_usd":0.015035,"input_uncached_usd":0.003095,"input_cached_usd":0.0,"output_usd":0.011940000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.127110915957019}
{"item_id":"senseval3.d000.s155.t004","gold_sense_keys":["have%2:40:00::"],"candidates":[{"index":1,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":2,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":3,"sense_key":"have%2:35:00::","synset_id":"take.v.35"},{"index":4,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":5,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":6,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":7,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":8,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":9,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":10,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":11,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":12,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":13,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":14,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":15,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":16,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":17,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":18,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":19,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d000.s155.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1050,"cached_input_tokens":0,"output_tokens":185,"reasoning_output_tokens":170},"cost":{"currency":"USD","total_usd":0.0108,"input_uncached_usd":0.00525,"input_cached_usd":0.0,"output_usd":0.00555,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.128381374990568}
{"item_id":"senseval3.d000.s155.t005","gold_sense_keys":["alibi%1:10:00::"],"candidates":[{"index":1,"sense_key":"alibi%1:10:00::","synset_id":"alibi.n.01"},{"index":2,"sense_key":"alibi%1:10:01::","synset_id":"excuse.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"alibi%1:10:00::","call_ids":["senseval3.d000.s155.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"alibi%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":328,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.00398,"input_uncached_usd":0.0016400000000000002,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4393087500939146}
{"item_id":"senseval3.d000.s156.t000","gold_sense_keys":["let%2:32:02::","let%2:41:00::"],"candidates":[{"index":1,"sense_key":"let%2:32:02::","synset_id":"permit.v.01"},{"index":2,"sense_key":"let%2:30:00::","synset_id":"let.v.05"},{"index":3,"sense_key":"let%2:41:00::","synset_id":"let.v.01"},{"index":4,"sense_key":"let%2:40:00::","synset_id":"lease.v.03"},{"index":5,"sense_key":"let%2:32:00::","synset_id":"let.v.02"},{"index":6,"sense_key":"let%2:30:01::","synset_id":"get.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"let%2:32:02::","call_ids":["senseval3.d000.s156.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"let%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":132},"cost":{"currency":"USD","total_usd":0.00689,"input_uncached_usd":0.0024500000000000004,"input_cached_usd":0.0,"output_usd":0.00444,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6439397499198094}
{"item_id":"senseval3.d000.s156.t001","gold_sense_keys":["prove%2:31:00::"],"candidates":[{"index":1,"sense_key":"prove%2:42:00::","synset_id":"prove.v.01"},{"index":2,"sense_key":"prove%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"prove%2:31:02::","synset_id":"prove.v.04"},{"index":4,"sense_key":"prove%2:32:00::","synset_id":"testify.v.02"},{"index":5,"sense_key":"prove%2:38:01::","synset_id":"rise.v.15"},{"index":6,"sense_key":"prove%2:41:03::","synset_id":"test.v.01"},{"index":7,"sense_key":"prove%2:36:00::","synset_id":"prove.v.08"},{"index":8,"sense_key":"prove%2:31:01::","synset_id":"prove.v.09"},{"index":9,"sense_key":"prove%2:38:00::","synset_id":"raise.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"prove%2:31:00::","call_ids":["senseval3.d000.s156.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"prove%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.009210000000000001,"input_uncached_usd":0.0029700000000000004,"input_cached_usd":0.0,"output_usd":0.00624,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.506456124945544}
{"item_id":"senseval3.d000.s157.t000","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":2,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":3,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":4,"sense_key":"think%2:30:00::","synset_id":"think.v.13"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":7,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":8,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":9,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":10,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":11,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":12,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":13,"sense_key":"think%2:31:01::","synset_id":"think.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval3.d000.s157.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":784,"cached_input_tokens":0,"output_tokens":1682,"reasoning_output_tokens":1647},"cost":{"currency":"USD","total_usd":0.054380000000000005,"input_uncached_usd":0.003920000000000001,"input_cached_usd":0.0,"output_usd":0.05046,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":24.770500082988292}
{"item_id":"senseval3.d000.s157.t002","gold_sense_keys":["favor%1:04:00::"],"candidates":[{"index":1,"sense_key":"favor%1:12:00::","synset_id":"favor.n.04"},{"index":2,"sense_key":"favor%1:06:00::","synset_id":"party_favor.n.01"},{"index":3,"sense_key":"favor%1:07:00::","synset_id":"favor.n.02"},{"index":4,"sense_key":"favor%1:04:00::","synset_id":"favor.n.01"},{"index":5,"sense_key":"favor%1:09:00::","synset_id":"favor.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"favor%1:04:00::","call_ids":["senseval3.d000.s157.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"favor%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":382,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00362,"input_uncached_usd":0.0019100000000000002,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6084014170337468}
{"item_id":"senseval3.d000.s158.t000","gold_sense_keys":["wait%2:42:00::"],"candidates":[{"index":1,"sense_key":"wait%2:42:01::","synset_id":"wait.v.02"},{"index":2,"sense_key":"wait%2:31:00::","synset_id":"expect.v.03"},{"index":3,"sense_key":"wait%2:41:00::","synset_id":"wait.v.04"},{"index":4,"sense_key":"wait%2:42:00::","synset_id":"wait.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"wait%2:42:00::","call_ids":["senseval3.d000.s158.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"wait%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":244,"reasoning_output_tokens":228},"cost":{"currency":"USD","total_usd":0.009320000000000002,"input_uncached_usd":0.002,"input_cached_usd":0.0,"output_usd":0.00732,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.154889459023252}
{"item_id":"senseval3.d000.s159.t001","gold_sense_keys":["liquor%1:13:00::"],"candidates":[{"index":1,"sense_key":"liquor%1:27:00::","synset_id":"liquor.n.02"},{"index":2,"sense_key":"liquor%1:13:00::","synset_id":"liquor.n.01"},{"index":3,"sense_key":"liquor%1:13:01::","synset_id":"liquor.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"liquor%1:13:00::","call_ids":["senseval3.d000.s159.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"liquor%1:13:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":346,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.0032,"input_uncached_usd":0.0017300000000000002,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3676966250641271}
{"item_id":"senseval3.d000.s159.t002","gold_sense_keys":["bottom%1:15:01::"],"candidates":[{"index":1,"sense_key":"bottom%1:15:01::","synset_id":"bottom.n.02"},{"index":2,"sense_key":"bottom%1:06:00::","synset_id":"bottom.n.07"},{"index":3,"sense_key":"bottom%1:08:00::","synset_id":"buttocks.n.01"},{"index":4,"sense_key":"bottom%1:28:00::","synset_id":"bottom.n.04"},{"index":5,"sense_key":"bottom%1:17:00::","synset_id":"bed.n.03"},{"index":6,"sense_key":"bottom%1:15:00::","synset_id":"bottom.n.01"},{"index":7,"sense_key":"bottom%1:17:01::","synset_id":"bottomland.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bottom%1:15:01::","call_ids":["senseval3.d000.s159.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bottom%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":95},"cost":{"currency":"USD","total_usd":0.005825,"input_uncached_usd":0.0024950000000000003,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2742374590598047}
{"item_id":"senseval3.d000.s159.t003","gold_sense_keys":["glass%1:06:00::"],"candidates":[{"index":1,"sense_key":"glass%1:06:04::","synset_id":"glass.n.07"},{"index":2,"sense_key":"glass%1:06:03::","synset_id":"field_glass.n.01"},{"index":3,"sense_key":"glass%1:23:00::","synset_id":"glass.n.03"},{"index":4,"sense_key":"glass%1:27:00::","synset_id":"glass.n.01"},{"index":5,"sense_key":"glass%1:06:00::","synset_id":"glass.n.02"},{"index":6,"sense_key":"glass%1:06:01::","synset_id":"methamphetamine.n.01"},{"index":7,"sense_key":"glass%1:06:02::","synset_id":"looking_glass.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"glass%1:06:00::","call_ids":["senseval3.d000.s159.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"glass%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.00392,"input_uncached_usd":0.0022700000000000003,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4625090420013294}
{"item_id":"senseval3.d001.s000.t000","gold_sense_keys":["rout%1:11:00::"],"candidates":[{"index":1,"sense_key":"rout%1:11:00::","synset_id":"rout.n.02"},{"index":2,"sense_key":"rout%1:14:00::","synset_id":"mob.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rout%1:11:00::","call_ids":["senseval3.d001.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rout%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":288,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.00309,"input_uncached_usd":0.00144,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6894576660124585}
{"item_id":"senseval3.d001.s000.t004","gold_sense_keys":["back%2:41:00::"],"candidates":[{"index":1,"sense_key":"back%2:42:00::","synset_id":"back.v.06"},{"index":2,"sense_key":"back%2:30:03::","synset_id":"back.v.08"},{"index":3,"sense_key":"back%2:41:01::","synset_id":"second.v.01"},{"index":4,"sense_key":"back%2:33:00::","synset_id":"bet_on.v.01"},{"index":5,"sense_key":"back%2:30:00::","synset_id":"back.v.09"},{"index":6,"sense_key":"back%2:30:01::","synset_id":"back.v.10"},{"index":7,"sense_key":"back%2:38:01::","synset_id":"back.v.04"},{"index":8,"sense_key":"back%2:40:00::","synset_id":"back.v.05"},{"index":9,"sense_key":"back%2:38:00::","synset_id":"back.v.02"},{"index":10,"sense_key":"back%2:41:00::","synset_id":"back.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"back%2:41:00::","call_ids":["senseval3.d001.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"back%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":608,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.005770000000000001,"input_uncached_usd":0.00304,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.701008375035599}
{"item_id":"senseval3.d001.s000.t005","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":376,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.00332,"input_uncached_usd":0.0018800000000000002,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.44095166597981}
{"item_id":"senseval3.d001.s000.t006","gold_sense_keys":["presidential%3:01:00::"],"candidates":[{"index":1,"sense_key":"presidential%3:00:00::","synset_id":"presidential.a.02"},{"index":2,"sense_key":"presidential%3:01:00::","synset_id":"presidential.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"presidential%3:01:00::","call_ids":["senseval3.d001.s000.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"presidential%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":306,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.0032700000000000003,"input_uncached_usd":0.0015300000000000001,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4917525000637397}
{"item_id":"senseval3.d001.s000.t007","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s000.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":307,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.003665,"input_uncached_usd":0.0015350000000000001,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.313193083042279}
{"item_id":"senseval3.d001.s000.t008","gold_sense_keys":["reminder%1:09:00::"],"candidates":[{"index":1,"sense_key":"reminder%1:10:00::","synset_id":"reminder.n.01"},{"index":2,"sense_key":"reminder%1:18:00::","synset_id":"admonisher.n.01"},{"index":3,"sense_key":"reminder%1:09:00::","synset_id":"reminder.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"reminder%1:09:00::","call_ids":["senseval3.d001.s000.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"reminder%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":335,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.0040750000000000005,"input_uncached_usd":0.001675,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9062199170002714}
{"item_id":"senseval3.d001.s000.t009","gold_sense_keys":["federal%5:00:00:national:01"],"candidates":[{"index":1,"sense_key":"federal%3:00:00::","synset_id":"federal.a.04"},{"index":2,"sense_key":"federal%3:01:02::","synset_id":"federal.a.02"},{"index":3,"sense_key":"federal%5:00:00:national:01","synset_id":"federal.s.01"},{"index":4,"sense_key":"federal%5:00:00:northern:02","synset_id":"union.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"federal%5:00:00:national:01","call_ids":["senseval3.d001.s000.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"federal%5:00:00:national:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":478,"reasoning_output_tokens":462},"cost":{"currency":"USD","total_usd":0.016575,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.01434,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.560714124934748}
{"item_id":"senseval3.d001.s000.t010","gold_sense_keys":["level%1:26:01::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":5,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":6,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":7,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":8,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"level%1:26:01::","call_ids":["senseval3.d001.s000.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"level%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":276,"reasoning_output_tokens":260},"cost":{"currency":"USD","total_usd":0.011275,"input_uncached_usd":0.0029950000000000003,"input_cached_usd":0.0,"output_usd":0.008280000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.6849320830078796}
{"item_id":"senseval3.d001.s000.t011","gold_sense_keys":["political%3:00:00::"],"candidates":[{"index":1,"sense_key":"political%3:01:02::","synset_id":"political.a.03"},{"index":2,"sense_key":"political%3:01:01::","synset_id":"political.a.02"},{"index":3,"sense_key":"political%3:00:00::","synset_id":"political.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"political%3:00:00::","call_ids":["senseval3.d001.s000.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"political%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":356,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":88},"cost":{"currency":"USD","total_usd":0.0049,"input_uncached_usd":0.0017800000000000001,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6566007080255076}
{"item_id":"senseval3.d001.s000.t013","gold_sense_keys":["past%3:00:00::"],"candidates":[{"index":1,"sense_key":"past%3:00:00::","synset_id":"past.a.01"},{"index":2,"sense_key":"past%5:00:00:outgoing:00","synset_id":"past.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"past%3:00:00::","call_ids":["senseval3.d001.s000.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"past%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":323,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.003295,"input_uncached_usd":0.0016150000000000001,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7520258750300854}
{"item_id":"senseval3.d001.s001.t000","gold_sense_keys":["presidential%3:01:00::"],"candidates":[{"index":1,"sense_key":"presidential%3:00:00::","synset_id":"presidential.a.02"},{"index":2,"sense_key":"presidential%3:01:00::","synset_id":"presidential.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"presidential%3:01:00::","call_ids":["senseval3.d001.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"presidential%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":321,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.003495,"input_uncached_usd":0.001605,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.733254292048514}
{"item_id":"senseval3.d001.s001.t001","gold_sense_keys":["election%1:04:01::"],"candidates":[{"index":1,"sense_key":"election%1:09:00::","synset_id":"election.n.04"},{"index":2,"sense_key":"election%1:04:01::","synset_id":"election.n.01"},{"index":3,"sense_key":"election%1:04:00::","synset_id":"election.n.02"},{"index":4,"sense_key":"election%1:26:00::","synset_id":"election.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"election%1:04:01::","call_ids":["senseval3.d001.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"election%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.00398,"input_uncached_usd":0.002,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6802300000563264}
{"item_id":"senseval3.d001.s001.t002","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:14:00::","synset_id":"class.n.06"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":4,"sense_key":"year%1:28:00::","synset_id":"year.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval3.d001.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":259,"reasoning_output_tokens":243},"cost":{"currency":"USD","total_usd":0.010015000000000001,"input_uncached_usd":0.002245,"input_cached_usd":0.0,"output_usd":0.00777,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9836696669226512}
{"item_id":"senseval3.d001.s001.t004","gold_sense_keys":["nation%1:14:00::"],"candidates":[{"index":1,"sense_key":"nation%1:14:01::","synset_id":"nation.n.02"},{"index":2,"sense_key":"nation%1:14:00::","synset_id":"state.n.04"},{"index":3,"sense_key":"nation%1:14:02::","synset_id":"nation.n.04"},{"index":4,"sense_key":"nation%1:18:00::","synset_id":"nation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"nation%1:14:00::","call_ids":["senseval3.d001.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"nation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":238,"reasoning_output_tokens":222},"cost":{"currency":"USD","total_usd":0.009365,"input_uncached_usd":0.002225,"input_cached_usd":0.0,"output_usd":0.0071400000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.245264583034441}
{"item_id":"senseval3.d001.s001.t007","gold_sense_keys":["choose%2:31:00::"],"candidates":[{"index":1,"sense_key":"choose%2:31:01::","synset_id":"choose.v.03"},{"index":2,"sense_key":"choose%2:31:00::","synset_id":"choose.v.01"},{"index":3,"sense_key":"choose%2:31:02::","synset_id":"choose.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"choose%2:31:00::","call_ids":["senseval3.d001.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"choose%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":412,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0179,"input_uncached_usd":0.00206,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.422395583009347}
{"item_id":"senseval3.d001.s001.t009","gold_sense_keys":["party%1:14:01::"],"candidates":[{"index":1,"sense_key":"party%1:14:00::","synset_id":"party.n.02"},{"index":2,"sense_key":"party%1:18:00::","synset_id":"party.n.05"},{"index":3,"sense_key":"party%1:14:01::","synset_id":"party.n.01"},{"index":4,"sense_key":"party%1:11:00::","synset_id":"party.n.04"},{"index":5,"sense_key":"party%1:14:02::","synset_id":"party.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"party%1:14:01::","call_ids":["senseval3.d001.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"party%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.00371,"input_uncached_usd":0.0022700000000000003,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.053385875071399}
{"item_id":"senseval3.d001.s001.t010","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"},{"index":2,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s001.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":322,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.00377,"input_uncached_usd":0.00161,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.684707541950047}
{"item_id":"senseval3.d001.s002.t000","gold_sense_keys":["percentage%1:24:00::"],"candidates":[{"index":1,"sense_key":"percentage%1:21:00::","synset_id":"share.n.01"},{"index":2,"sense_key":"percentage%1:24:00::","synset_id":"percentage.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"percentage%1:24:00::","call_ids":["senseval3.d001.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"percentage%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":363,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.0033750000000000004,"input_uncached_usd":0.0018150000000000002,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.769148625084199}
{"item_id":"senseval3.d001.s002.t001","gold_sense_keys":["equal%2:42:00::"],"candidates":[{"index":1,"sense_key":"equal%2:42:00::","synset_id":"equal.v.01"},{"index":2,"sense_key":"equal%2:42:01::","synset_id":"equal.v.02"},{"index":3,"sense_key":"equal%2:30:00::","synset_id":"equal.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"equal%2:42:00::","call_ids":["senseval3.d001.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"equal%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0038450000000000003,"input_uncached_usd":0.002135,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.62395054101944}
{"item_id":"senseval3.d001.s003.t000","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":2,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":3,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":4,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":5,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":6,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":7,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":8,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":9,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":10,"sense_key":"know%2:31:14::","synset_id":"know.v.11"},{"index":11,"sense_key":"know%2:31:02::","synset_id":"know.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval3.d001.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":835,"cached_input_tokens":0,"output_tokens":372,"reasoning_output_tokens":356},"cost":{"currency":"USD","total_usd":0.015335000000000001,"input_uncached_usd":0.004175000000000001,"input_cached_usd":0.0,"output_usd":0.01116,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.176918040961027}
{"item_id":"senseval3.d001.s003.t002","gold_sense_keys":["tend%2:42:01::"],"candidates":[{"index":1,"sense_key":"tend%2:42:01::","synset_id":"tend.v.01"},{"index":2,"sense_key":"tend%2:41:00::","synset_id":"tend.v.02"},{"index":3,"sense_key":"tend%2:41:01::","synset_id":"tend.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tend%2:42:01::","call_ids":["senseval3.d001.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tend%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":436,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.0044,"input_uncached_usd":0.00218,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.149457500083372}
{"item_id":"senseval3.d001.s003.t003","gold_sense_keys":["favor%2:41:00::"],"candidates":[{"index":1,"sense_key":"favor%2:41:03::","synset_id":"privilege.v.01"},{"index":2,"sense_key":"favor%2:41:00::","synset_id":"prefer.v.03"},{"index":3,"sense_key":"favor%2:31:00::","synset_id":"favor.v.02"},{"index":4,"sense_key":"favor%2:41:01::","synset_id":"favor.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"favor%2:41:00::","call_ids":["senseval3.d001.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"favor%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":111},"cost":{"currency":"USD","total_usd":0.006025000000000001,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.00381,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.793412958038971}
{"item_id":"senseval3.d001.s003.t004","gold_sense_keys":["republican%1:18:01::"],"candidates":[{"index":1,"sense_key":"republican%1:18:01::","synset_id":"republican.n.01"},{"index":2,"sense_key":"republican%1:17:00::","synset_id":"republican.n.03"},{"index":3,"sense_key":"republican%1:18:00::","synset_id":"republican.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%1:18:01::","call_ids":["senseval3.d001.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.0037900000000000004,"input_uncached_usd":0.0020800000000000003,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.7022425420582294}
{"item_id":"senseval3.d001.s003.t005","gold_sense_keys":["race%1:11:01::"],"candidates":[{"index":1,"sense_key":"race%1:11:00::","synset_id":"race.n.02"},{"index":2,"sense_key":"race%1:19:00::","synset_id":"slipstream.n.01"},{"index":3,"sense_key":"race%1:11:01::","synset_id":"race.n.01"},{"index":4,"sense_key":"race%1:14:00::","synset_id":"race.n.03"},{"index":5,"sense_key":"race%1:14:01::","synset_id":"subspecies.n.01"},{"index":6,"sense_key":"race%1:06:00::","synset_id":"raceway.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"race%1:11:01::","call_ids":["senseval3.d001.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"race%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.005175000000000001,"input_uncached_usd":0.0027150000000000004,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8984958750661463}
{"item_id":"senseval3.d001.s003.t006","gold_sense_keys":["president%1:04:04::"],"candidates":[{"index":1,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":2,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":3,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":4,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":5,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"president%1:04:04::","call_ids":["senseval3.d001.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"president%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":561,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018645000000000002,"input_uncached_usd":0.002805,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.949956625001505}
{"item_id":"senseval3.d001.s004.t000","gold_sense_keys":["presidential%3:01:00::"],"candidates":[{"index":1,"sense_key":"presidential%3:00:00::","synset_id":"presidential.a.02"},{"index":2,"sense_key":"presidential%3:01:00::","synset_id":"presidential.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"presidential%3:01:00::","call_ids":["senseval3.d001.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"presidential%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.00366,"input_uncached_usd":0.00201,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4879982919665053}
{"item_id":"senseval3.d001.s004.t001","gold_sense_keys":["election%1:04:01::"],"candidates":[{"index":1,"sense_key":"election%1:04:01::","synset_id":"election.n.01"},{"index":2,"sense_key":"election%1:09:00::","synset_id":"election.n.04"},{"index":3,"sense_key":"election%1:26:00::","synset_id":"election.n.03"},{"index":4,"sense_key":"election%1:04:00::","synset_id":"election.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"election%1:04:01::","call_ids":["senseval3.d001.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"election%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.004175,"input_uncached_usd":0.002405,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9167792500229552}
{"item_id":"senseval3.d001.s004.t002","gold_sense_keys":["past%3:00:00::"],"candidates":[{"index":1,"sense_key":"past%5:00:00:outgoing:00","synset_id":"past.s.01"},{"index":2,"sense_key":"past%3:00:00::","synset_id":"past.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"past%3:00:00::","call_ids":["senseval3.d001.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"past%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.004225,"input_uncached_usd":0.002095,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.876256457995623}
{"item_id":"senseval3.d001.s004.t004","gold_sense_keys":["presidential%3:01:00::"],"candidates":[{"index":1,"sense_key":"presidential%3:00:00::","synset_id":"presidential.a.02"},{"index":2,"sense_key":"presidential%3:01:00::","synset_id":"presidential.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"presidential%3:01:00::","call_ids":["senseval3.d001.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"presidential%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.00453,"input_uncached_usd":0.00201,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0215513750445098}
{"item_id":"senseval3.d001.s004.t007","gold_sense_keys":["percentage%1:24:00::"],"candidates":[{"index":1,"sense_key":"percentage%1:24:00::","synset_id":"percentage.n.01"},{"index":2,"sense_key":"percentage%1:21:00::","synset_id":"share.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"percentage%1:24:00::","call_ids":["senseval3.d001.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"percentage%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.00371,"input_uncached_usd":0.0021200000000000004,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4121124169323593}
{"item_id":"senseval3.d001.s004.t008","gold_sense_keys":["major%3:00:01::","major%3:00:06::"],"candidates":[{"index":1,"sense_key":"major%3:00:03::","synset_id":"major.a.07"},{"index":2,"sense_key":"major%3:00:06::","synset_id":"major.a.01"},{"index":3,"sense_key":"major%3:00:05::","synset_id":"major.a.04"},{"index":4,"sense_key":"major%3:00:01::","synset_id":"major.a.03"},{"index":5,"sense_key":"major%3:00:04::","synset_id":"major.a.05"},{"index":6,"sense_key":"major%3:00:02::","synset_id":"major.a.02"},{"index":7,"sense_key":"major%5:00:00:senior:00","synset_id":"major.s.01"},{"index":8,"sense_key":"major%3:00:07::","synset_id":"major.a.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"major%3:00:06::","call_ids":["senseval3.d001.s004.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"major%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":363,"reasoning_output_tokens":348},"cost":{"currency":"USD","total_usd":0.013875,"input_uncached_usd":0.0029850000000000002,"input_cached_usd":0.0,"output_usd":0.01089,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.5062540830113}
{"item_id":"senseval3.d001.s004.t009","gold_sense_keys":["party%1:14:01::"],"candidates":[{"index":1,"sense_key":"party%1:11:00::","synset_id":"party.n.04"},{"index":2,"sense_key":"party%1:14:00::","synset_id":"party.n.02"},{"index":3,"sense_key":"party%1:14:02::","synset_id":"party.n.03"},{"index":4,"sense_key":"party%1:14:01::","synset_id":"party.n.01"},{"index":5,"sense_key":"party%1:18:00::","synset_id":"party.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"party%1:14:01::","call_ids":["senseval3.d001.s004.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"party%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.004325000000000001,"input_uncached_usd":0.0026750000000000003,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5199147089151666}
{"item_id":"senseval3.d001.s004.t010","gold_sense_keys":["popular%5:00:00:democratic:00"],"candidates":[{"index":1,"sense_key":"popular%5:00:00:nonclassical:00","synset_id":"popular.s.03"},{"index":2,"sense_key":"popular%5:00:01:common:01","synset_id":"democratic.s.01"},{"index":3,"sense_key":"popular%5:00:00:democratic:00","synset_id":"popular.s.01"},{"index":4,"sense_key":"popular%3:00:00::","synset_id":"popular.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"popular%5:00:00:democratic:00","call_ids":["senseval3.d001.s004.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"popular%5:00:00:democratic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004555,"input_uncached_usd":0.002575,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.1939667500555515}
{"item_id":"senseval3.d001.s004.t011","gold_sense_keys":["vote%1:09:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":2,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":3,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":4,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":5,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vote%1:04:00::","call_ids":["senseval3.d001.s004.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vote%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01868,"input_uncached_usd":0.00284,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.838555374997668}
{"item_id":"senseval3.d001.s004.t012","gold_sense_keys":["president%1:04:04::"],"candidates":[{"index":1,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":2,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"},{"index":3,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":4,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":5,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":6,"sense_key":"president%1:18:00::","synset_id":"president.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"president%1:04:04::","call_ids":["senseval3.d001.s004.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"president%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":210,"reasoning_output_tokens":194},"cost":{"currency":"USD","total_usd":0.009170000000000001,"input_uncached_usd":0.00287,"input_cached_usd":0.0,"output_usd":0.0063,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.501096124993637}
{"item_id":"senseval3.d001.s004.t014","gold_sense_keys":["seat%1:07:00::"],"candidates":[{"index":1,"sense_key":"seat%1:15:01::","synset_id":"seat.n.01"},{"index":2,"sense_key":"seat%1:15:00::","synset_id":"seat.n.05"},{"index":3,"sense_key":"seat%1:06:01::","synset_id":"seat.n.03"},{"index":4,"sense_key":"seat%1:06:03::","synset_id":"seat.n.08"},{"index":5,"sense_key":"seat%1:08:00::","synset_id":"buttocks.n.01"},{"index":6,"sense_key":"seat%1:06:02::","synset_id":"seat.n.09"},{"index":7,"sense_key":"seat%1:15:02::","synset_id":"seat.n.06"},{"index":8,"sense_key":"seat%1:06:00::","synset_id":"seat.n.04"},{"index":9,"sense_key":"seat%1:07:00::","synset_id":"seat.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"seat%1:07:00::","call_ids":["senseval3.d001.s004.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"seat%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.005445,"input_uncached_usd":0.003615,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.681562041048892}
{"item_id":"senseval3.d001.s004.t015","gold_sense_keys":["popular%5:00:00:democratic:00"],"candidates":[{"index":1,"sense_key":"popular%5:00:01:common:01","synset_id":"democratic.s.01"},{"index":2,"sense_key":"popular%5:00:00:democratic:00","synset_id":"popular.s.01"},{"index":3,"sense_key":"popular%3:00:00::","synset_id":"popular.a.01"},{"index":4,"sense_key":"popular%5:00:00:nonclassical:00","synset_id":"popular.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"popular%5:00:00:democratic:00","call_ids":["senseval3.d001.s004.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"popular%5:00:00:democratic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":94},"cost":{"currency":"USD","total_usd":0.005875,"input_uncached_usd":0.002575,"input_cached_usd":0.0,"output_usd":0.0033,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2889012079685926}
{"item_id":"senseval3.d001.s004.t016","gold_sense_keys":["vote%1:09:00::"],"candidates":[{"index":1,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"},{"index":2,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":3,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":4,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":5,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"vote%1:04:00::","call_ids":["senseval3.d001.s004.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"vote%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":232,"reasoning_output_tokens":217},"cost":{"currency":"USD","total_usd":0.0098,"input_uncached_usd":0.00284,"input_cached_usd":0.0,"output_usd":0.00696,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.949409292079508}
{"item_id":"senseval3.d001.s005.t001","gold_sense_keys":["opposite%1:24:03::"],"candidates":[{"index":1,"sense_key":"opposite%1:10:00::","synset_id":"antonym.n.01"},{"index":2,"sense_key":"opposite%1:18:00::","synset_id":"opposition.n.04"},{"index":3,"sense_key":"opposite%1:24:03::","synset_id":"reverse.n.01"},{"index":4,"sense_key":"opposite%1:24:04::","synset_id":"inverse.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"opposite%1:24:03::","call_ids":["senseval3.d001.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"opposite%1:24:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":245,"reasoning_output_tokens":229},"cost":{"currency":"USD","total_usd":0.00999,"input_uncached_usd":0.0026400000000000004,"input_cached_usd":0.0,"output_usd":0.00735,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.596832292038016}
{"item_id":"senseval3.d001.s006.t000","gold_sense_keys":["account_for%2:42:00::"],"candidates":[{"index":1,"sense_key":"account_for%2:32:00::","synset_id":"account_for.v.02"},{"index":2,"sense_key":"account_for%2:42:00::","synset_id":"account_for.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"account_for%2:42:00::","call_ids":["senseval3.d001.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"account_for%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":384,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.004620000000000001,"input_uncached_usd":0.0019200000000000003,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.927784916944802}
{"item_id":"senseval3.d001.s006.t001","gold_sense_keys":["result%1:11:00::"],"candidates":[{"index":1,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":2,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":3,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"},{"index":4,"sense_key":"result%1:11:00::","synset_id":"result.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"result%1:11:00::","call_ids":["senseval3.d001.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"result%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":203,"reasoning_output_tokens":187},"cost":{"currency":"USD","total_usd":0.008715,"input_uncached_usd":0.002625,"input_cached_usd":0.0,"output_usd":0.00609,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.824350417009555}
{"item_id":"senseval3.d001.s006.t002","gold_sense_keys":["recent%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"recent%5:00:00:past:00","synset_id":"late.s.02"},{"index":2,"sense_key":"recent%5:00:00:new:00","synset_id":"recent.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recent%5:00:00:past:00","call_ids":["senseval3.d001.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recent%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":391,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.004325000000000001,"input_uncached_usd":0.001955,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.8680864999769256}
{"item_id":"senseval3.d001.s006.t003","gold_sense_keys":["decade%1:28:00::"],"candidates":[{"index":1,"sense_key":"decade%1:28:00::","synset_id":"decade.n.01"},{"index":2,"sense_key":"decade%1:23:00::","synset_id":"ten.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decade%1:28:00::","call_ids":["senseval3.d001.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decade%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":395,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.003385,"input_uncached_usd":0.001975,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5687452499987558}
{"item_id":"senseval3.d001.s007.t000","gold_sense_keys":["simple%3:00:02::","simple%5:00:00:easy:01"],"candidates":[{"index":1,"sense_key":"simple%3:00:02::","synset_id":"simple.a.01"},{"index":2,"sense_key":"simple%5:00:02:plain:01","synset_id":"bare.s.04"},{"index":3,"sense_key":"simple%5:00:00:naive:00","synset_id":"childlike.s.02"},{"index":4,"sense_key":"simple%3:00:01::","synset_id":"simple.a.06"},{"index":5,"sense_key":"simple%5:00:00:retarded:00","synset_id":"dim-witted.s.01"},{"index":6,"sense_key":"simple%5:00:00:plain:01","synset_id":"simple.s.05"},{"index":7,"sense_key":"simple%5:00:00:easy:01","synset_id":"elementary.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"simple%3:00:02::","call_ids":["senseval3.d001.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"simple%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":160,"reasoning_output_tokens":144},"cost":{"currency":"USD","total_usd":0.00774,"input_uncached_usd":0.0029400000000000003,"input_cached_usd":0.0,"output_usd":0.0048000000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8536438749870285}
{"item_id":"senseval3.d001.s007.t004","gold_sense_keys":["partial%5:00:00:incomplete:00"],"candidates":[{"index":1,"sense_key":"partial%3:00:01::","synset_id":"partial.a.02"},{"index":2,"sense_key":"partial%5:00:00:inclined:02","synset_id":"fond.s.03"},{"index":3,"sense_key":"partial%5:00:00:incomplete:00","synset_id":"partial.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"partial%5:00:00:incomplete:00","call_ids":["senseval3.d001.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"partial%5:00:00:incomplete:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":398,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.00334,"input_uncached_usd":0.00199,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5026784999063239}
{"item_id":"senseval3.d001.s007.t007","gold_sense_keys":["display%2:39:00::"],"candidates":[{"index":1,"sense_key":"display%2:39:00::","synset_id":"expose.v.03"},{"index":2,"sense_key":"display%2:41:00::","synset_id":"display.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"display%2:39:00::","call_ids":["senseval3.d001.s007.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"display%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":371,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.004375,"input_uncached_usd":0.001855,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4095424170373008}
{"item_id":"senseval3.d001.s007.t008","gold_sense_keys":["american%1:18:00::"],"candidates":[{"index":1,"sense_key":"american%1:18:01::","synset_id":"american.n.03"},{"index":2,"sense_key":"american%1:18:00::","synset_id":"american.n.01"},{"index":3,"sense_key":"american%1:10:00::","synset_id":"american_english.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"american%1:18:00::","call_ids":["senseval3.d001.s007.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"american%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":375,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.003705,"input_uncached_usd":0.0018750000000000001,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6135221659205854}
{"item_id":"senseval3.d001.s008.t002","gold_sense_keys":["assumption%1:09:00::","assumption%1:10:00::"],"candidates":[{"index":1,"sense_key":"assumption%1:04:01::","synset_id":"assumption.n.07"},{"index":2,"sense_key":"assumption%1:10:00::","synset_id":"premise.n.01"},{"index":3,"sense_key":"assumption%1:04:00::","synset_id":"assumption.n.03"},{"index":4,"sense_key":"assumption%1:07:00::","synset_id":"presumption.n.03"},{"index":5,"sense_key":"assumption%1:11:00::","synset_id":"assumption.n.05"},{"index":6,"sense_key":"assumption%1:09:00::","synset_id":"assumption.n.02"},{"index":7,"sense_key":"assumption%1:28:00::","synset_id":"assumption.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"assumption%1:10:00::","call_ids":["senseval3.d001.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"assumption%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019335,"input_uncached_usd":0.0034950000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.754262833041139}
{"item_id":"senseval3.d001.s009.t002","gold_sense_keys":["brand%1:09:00::"],"candidates":[{"index":1,"sense_key":"brand%1:10:02::","synset_id":"brand.n.03"},{"index":2,"sense_key":"brand%1:10:00::","synset_id":"trade_name.n.01"},{"index":3,"sense_key":"brand%1:10:01::","synset_id":"mark.n.06"},{"index":4,"sense_key":"brand%1:27:00::","synset_id":"brand.n.04"},{"index":5,"sense_key":"brand%1:09:00::","synset_id":"brand.n.02"},{"index":6,"sense_key":"brand%1:06:00::","synset_id":"sword.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"brand%1:09:00::","call_ids":["senseval3.d001.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"brand%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018935,"input_uncached_usd":0.003095,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.162431625067256}
{"item_id":"senseval3.d001.s009.t004","gold_sense_keys":["political%3:00:00::","political%3:01:02::"],"candidates":[{"index":1,"sense_key":"political%3:01:01::","synset_id":"political.a.02"},{"index":2,"sense_key":"political%3:00:00::","synset_id":"political.a.01"},{"index":3,"sense_key":"political%3:01:02::","synset_id":"political.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"political%3:00:00::","call_ids":["senseval3.d001.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"political%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":407,"reasoning_output_tokens":391},"cost":{"currency":"USD","total_usd":0.01477,"input_uncached_usd":0.00256,"input_cached_usd":0.0,"output_usd":0.01221,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.565886833006516}
{"item_id":"senseval3.d001.s009.t005","gold_sense_keys":["agent%1:18:02::"],"candidates":[{"index":1,"sense_key":"agent%1:18:03::","synset_id":"agent.n.05"},{"index":2,"sense_key":"agent%1:17:00::","synset_id":"agent.n.01"},{"index":3,"sense_key":"agent%1:27:00::","synset_id":"agent.n.03"},{"index":4,"sense_key":"agent%1:10:00::","synset_id":"agentive_role.n.01"},{"index":5,"sense_key":"agent%1:18:01::","synset_id":"agent.n.04"},{"index":6,"sense_key":"agent%1:18:02::","synset_id":"agent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"agent%1:18:02::","call_ids":["senseval3.d001.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"agent%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.005505,"input_uncached_usd":0.0029850000000000002,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.07812120905146}
{"item_id":"senseval3.d001.s009.t006","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00436,"input_uncached_usd":0.0025900000000000003,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.766021916992031}
{"item_id":"senseval3.d001.s009.t007","gold_sense_keys":["brand%1:09:00::"],"candidates":[{"index":1,"sense_key":"brand%1:10:00::","synset_id":"trade_name.n.01"},{"index":2,"sense_key":"brand%1:10:01::","synset_id":"mark.n.06"},{"index":3,"sense_key":"brand%1:10:02::","synset_id":"brand.n.03"},{"index":4,"sense_key":"brand%1:09:00::","synset_id":"brand.n.02"},{"index":5,"sense_key":"brand%1:06:00::","synset_id":"sword.n.01"},{"index":6,"sense_key":"brand%1:27:00::","synset_id":"brand.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"brand%1:09:00::","call_ids":["senseval3.d001.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"brand%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":342,"reasoning_output_tokens":326},"cost":{"currency":"USD","total_usd":0.013355,"input_uncached_usd":0.003095,"input_cached_usd":0.0,"output_usd":0.01026,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.458648834028281}
{"item_id":"senseval3.d001.s009.t009","gold_sense_keys":["minimalist%5:00:00:moderate:00"],"candidates":[{"index":1,"sense_key":"minimalist%5:00:00:moderate:00","synset_id":"minimalist.s.01"},{"index":2,"sense_key":"minimalist%3:01:00::","synset_id":"minimalist.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"minimalist%5:00:00:moderate:00","call_ids":["senseval3.d001.s009.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"minimalist%5:00:00:moderate:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":81},"cost":{"currency":"USD","total_usd":0.005145,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.00288,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3866188330575824}
{"item_id":"senseval3.d001.s009.t010","gold_sense_keys":["state%1:14:00::","state%1:14:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":3,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":4,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":5,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":6,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"},{"index":7,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":8,"sense_key":"state%1:26:01::","synset_id":"state.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"state%1:14:01::","call_ids":["senseval3.d001.s009.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"state%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":786,"cached_input_tokens":0,"output_tokens":301,"reasoning_output_tokens":285},"cost":{"currency":"USD","total_usd":0.01296,"input_uncached_usd":0.00393,"input_cached_usd":0.0,"output_usd":0.00903,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.226797958021052}
{"item_id":"senseval3.d001.s009.t011","gold_sense_keys":["virtue%1:07:03::"],"candidates":[{"index":1,"sense_key":"virtue%1:07:04::","synset_id":"virtue.n.04"},{"index":2,"sense_key":"virtue%1:07:03::","synset_id":"merit.n.01"},{"index":3,"sense_key":"virtue%1:07:01::","synset_id":"virtue.n.01"},{"index":4,"sense_key":"virtue%1:07:02::","synset_id":"virtue.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"virtue%1:07:03::","call_ids":["senseval3.d001.s009.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"virtue%1:07:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":99},"cost":{"currency":"USD","total_usd":0.0060750000000000005,"input_uncached_usd":0.002625,"input_cached_usd":0.0,"output_usd":0.00345,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5869775420287624}
{"item_id":"senseval3.d001.s009.t012","gold_sense_keys":["private%3:00:00::"],"candidates":[{"index":1,"sense_key":"private%5:00:00:inward:00","synset_id":"secret.s.05"},{"index":2,"sense_key":"private%5:00:00:personal:00","synset_id":"individual.s.03"},{"index":3,"sense_key":"private%3:00:00::","synset_id":"private.a.01"},{"index":4,"sense_key":"private%5:00:02:personal:00","synset_id":"private.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"private%3:00:00::","call_ids":["senseval3.d001.s009.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"private%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018565,"input_uncached_usd":0.0027250000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.412141332984902}
{"item_id":"senseval3.d001.s009.t013","gold_sense_keys":["market%1:04:00::"],"candidates":[{"index":1,"sense_key":"market%1:06:01::","synset_id":"marketplace.n.02"},{"index":2,"sense_key":"market%1:06:00::","synset_id":"grocery_store.n.01"},{"index":3,"sense_key":"market%1:14:00::","synset_id":"market.n.02"},{"index":4,"sense_key":"market%1:14:01::","synset_id":"market.n.04"},{"index":5,"sense_key":"market%1:04:00::","synset_id":"market.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"market%1:04:00::","call_ids":["senseval3.d001.s009.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"market%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.005215000000000001,"input_uncached_usd":0.0030550000000000004,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9099921670276672}
{"item_id":"senseval3.d001.s009.t014","gold_sense_keys":["vice%1:07:00::"],"candidates":[{"index":1,"sense_key":"vice%1:07:00::","synset_id":"frailty.n.02"},{"index":2,"sense_key":"vice%1:04:00::","synset_id":"vice.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vice%1:07:00::","call_ids":["senseval3.d001.s009.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vice%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":261,"reasoning_output_tokens":246},"cost":{"currency":"USD","total_usd":0.010100000000000001,"input_uncached_usd":0.0022700000000000003,"input_cached_usd":0.0,"output_usd":0.00783,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.9367584170540795}
{"item_id":"senseval3.d001.s009.t015","gold_sense_keys":["public%3:00:00::"],"candidates":[{"index":1,"sense_key":"public%5:00:00:common:02","synset_id":"public.s.01"},{"index":2,"sense_key":"public%3:00:00::","synset_id":"public.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"public%3:00:00::","call_ids":["senseval3.d001.s009.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"public%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":184,"reasoning_output_tokens":168},"cost":{"currency":"USD","total_usd":0.00787,"input_uncached_usd":0.00235,"input_cached_usd":0.0,"output_usd":0.00552,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.3338368750410154}
{"item_id":"senseval3.d001.s009.t016","gold_sense_keys":["action%1:04:04::"],"candidates":[{"index":1,"sense_key":"action%1:04:00::","synset_id":"action.n.10"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":4,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":5,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":6,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":7,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":8,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":9,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":10,"sense_key":"action%1:04:02::","synset_id":"action.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"action%1:04:04::","call_ids":["senseval3.d001.s009.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"action%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":850,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":110},"cost":{"currency":"USD","total_usd":0.00803,"input_uncached_usd":0.00425,"input_cached_usd":0.0,"output_usd":0.00378,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5635803330224007}
{"item_id":"senseval3.d001.s009.t017","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s009.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00443,"input_uncached_usd":0.00266,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.691664541955106}
{"item_id":"senseval3.d001.s009.t018","gold_sense_keys":["brand%1:09:00::"],"candidates":[{"index":1,"sense_key":"brand%1:10:00::","synset_id":"trade_name.n.01"},{"index":2,"sense_key":"brand%1:27:00::","synset_id":"brand.n.04"},{"index":3,"sense_key":"brand%1:09:00::","synset_id":"brand.n.02"},{"index":4,"sense_key":"brand%1:10:02::","synset_id":"brand.n.03"},{"index":5,"sense_key":"brand%1:06:00::","synset_id":"sword.n.01"},{"index":6,"sense_key":"brand%1:10:01::","synset_id":"mark.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"brand%1:09:00::","call_ids":["senseval3.d001.s009.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"brand%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":160,"reasoning_output_tokens":144},"cost":{"currency":"USD","total_usd":0.007895000000000001,"input_uncached_usd":0.003095,"input_cached_usd":0.0,"output_usd":0.0048000000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5006071670213714}
{"item_id":"senseval3.d001.s009.t022","gold_sense_keys":["public%3:00:00::"],"candidates":[{"index":1,"sense_key":"public%3:00:00::","synset_id":"public.a.01"},{"index":2,"sense_key":"public%5:00:00:common:02","synset_id":"public.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"public%3:00:00::","call_ids":["senseval3.d001.s009.t022__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"public%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":113},"cost":{"currency":"USD","total_usd":0.00622,"input_uncached_usd":0.00235,"input_cached_usd":0.0,"output_usd":0.00387,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.57807362510357}
{"item_id":"senseval3.d001.s009.t023","gold_sense_keys":["intervention%1:04:00::"],"candidates":[{"index":1,"sense_key":"intervention%1:04:00::","synset_id":"intervention.n.01"},{"index":2,"sense_key":"intervention%1:04:02::","synset_id":"treatment.n.01"},{"index":3,"sense_key":"intervention%1:10:00::","synset_id":"intervention.n.02"},{"index":4,"sense_key":"intervention%1:04:01::","synset_id":"intervention.n.04"},{"index":5,"sense_key":"intervention%1:04:03::","synset_id":"interposition.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"intervention%1:04:00::","call_ids":["senseval3.d001.s009.t023__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"intervention%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":121},"cost":{"currency":"USD","total_usd":0.00725,"input_uncached_usd":0.0031400000000000004,"input_cached_usd":0.0,"output_usd":0.00411,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.424846916925162}
{"item_id":"senseval3.d001.s009.t024","gold_sense_keys":["remedy%2:30:00::"],"candidates":[{"index":1,"sense_key":"remedy%2:30:00::","synset_id":"rectify.v.04"},{"index":2,"sense_key":"remedy%2:29:00::","synset_id":"remedy.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"remedy%2:30:00::","call_ids":["senseval3.d001.s009.t024__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"remedy%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.004805,"input_uncached_usd":0.0023750000000000004,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.61231383390259}
{"item_id":"senseval3.d001.s009.t025","gold_sense_keys":["excess%1:07:00::"],"candidates":[{"index":1,"sense_key":"excess%1:04:00::","synset_id":"overindulgence.n.01"},{"index":2,"sense_key":"excess%1:26:00::","synset_id":"surfeit.n.01"},{"index":3,"sense_key":"excess%1:07:02::","synset_id":"excess.n.01"},{"index":4,"sense_key":"excess%1:07:00::","synset_id":"excess.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"excess%1:07:00::","call_ids":["senseval3.d001.s009.t025__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"excess%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":115},"cost":{"currency":"USD","total_usd":0.006670000000000001,"input_uncached_usd":0.0027400000000000002,"input_cached_usd":0.0,"output_usd":0.00393,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.297849209047854}
{"item_id":"senseval3.d001.s009.t026","gold_sense_keys":["attendant%5:00:00:subsequent:00"],"candidates":[{"index":1,"sense_key":"attendant%5:00:00:present:02","synset_id":"attendant.s.01"},{"index":2,"sense_key":"attendant%5:00:00:subsequent:00","synset_id":"attendant.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"attendant%5:00:00:subsequent:00","call_ids":["senseval3.d001.s009.t026__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"attendant%5:00:00:subsequent:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0044150000000000005,"input_uncached_usd":0.0025250000000000003,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7005601250566542}
{"item_id":"senseval3.d001.s009.t027","gold_sense_keys":["pursuit%1:09:00::"],"candidates":[{"index":1,"sense_key":"pursuit%1:04:01::","synset_id":"pastime.n.01"},{"index":2,"sense_key":"pursuit%1:04:00::","synset_id":"pursuit.n.01"},{"index":3,"sense_key":"pursuit%1:04:02::","synset_id":"avocation.n.01"},{"index":4,"sense_key":"pursuit%1:09:00::","synset_id":"pursuit.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"pursuit%1:09:00::","call_ids":["senseval3.d001.s009.t027__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"pursuit%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":96},"cost":{"currency":"USD","total_usd":0.006315,"input_uncached_usd":0.002955,"input_cached_usd":0.0,"output_usd":0.00336,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4462538339430466}
{"item_id":"senseval3.d001.s009.t028","gold_sense_keys":["private%3:00:00::"],"candidates":[{"index":1,"sense_key":"private%5:00:00:inward:00","synset_id":"secret.s.05"},{"index":2,"sense_key":"private%5:00:00:personal:00","synset_id":"individual.s.03"},{"index":3,"sense_key":"private%3:00:00::","synset_id":"private.a.01"},{"index":4,"sense_key":"private%5:00:02:personal:00","synset_id":"private.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"private%5:00:00:personal:00","call_ids":["senseval3.d001.s009.t028__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"private%5:00:00:personal:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":366,"reasoning_output_tokens":350},"cost":{"currency":"USD","total_usd":0.013705,"input_uncached_usd":0.0027250000000000004,"input_cached_usd":0.0,"output_usd":0.01098,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.4116023749811575}
{"item_id":"senseval3.d001.s009.t029","gold_sense_keys":["interest%1:07:01::"],"candidates":[{"index":1,"sense_key":"interest%1:21:03::","synset_id":"interest.n.05"},{"index":2,"sense_key":"interest%1:14:00::","synset_id":"interest.n.06"},{"index":3,"sense_key":"interest%1:04:01::","synset_id":"pastime.n.01"},{"index":4,"sense_key":"interest%1:07:02::","synset_id":"interest.n.03"},{"index":5,"sense_key":"interest%1:09:00::","synset_id":"interest.n.01"},{"index":6,"sense_key":"interest%1:21:00::","synset_id":"interest.n.04"},{"index":7,"sense_key":"interest%1:07:01::","synset_id":"sake.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"interest%1:07:01::","call_ids":["senseval3.d001.s009.t029__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"interest%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":722,"cached_input_tokens":0,"output_tokens":519,"reasoning_output_tokens":503},"cost":{"currency":"USD","total_usd":0.019180000000000003,"input_uncached_usd":0.0036100000000000004,"input_cached_usd":0.0,"output_usd":0.01557,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.158441124949604}
{"item_id":"senseval3.d001.s010.t001","gold_sense_keys":["representative%1:18:01::"],"candidates":[{"index":1,"sense_key":"representative%1:18:00::","synset_id":"representative.n.01"},{"index":2,"sense_key":"representative%1:18:02::","synset_id":"spokesperson.n.01"},{"index":3,"sense_key":"representative%1:09:00::","synset_id":"example.n.01"},{"index":4,"sense_key":"representative%1:18:01::","synset_id":"congressman.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"representative%1:18:01::","call_ids":["senseval3.d001.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"representative%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018595,"input_uncached_usd":0.002755,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.95277808397077}
{"item_id":"senseval3.d001.s010.t002","gold_sense_keys":["have%2:40:00::","have%2:42:12::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":3,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":4,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":5,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":6,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":7,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":10,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":11,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":12,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":13,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":14,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":15,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":16,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":17,"sense_key":"have%2:35:00::","synset_id":"take.v.35"},{"index":18,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":19,"sense_key":"have%2:40:01::","synset_id":"have.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d001.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1217,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":132},"cost":{"currency":"USD","total_usd":0.010495,"input_uncached_usd":0.006085,"input_cached_usd":0.0,"output_usd":0.00441,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.807948541943915}
{"item_id":"senseval3.d001.s010.t004","gold_sense_keys":["responsibility%1:04:00::"],"candidates":[{"index":1,"sense_key":"responsibility%1:04:00::","synset_id":"duty.n.01"},{"index":2,"sense_key":"responsibility%1:26:00::","synset_id":"province.n.02"},{"index":3,"sense_key":"responsibility%1:07:00::","synset_id":"responsibility.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"responsibility%1:04:00::","call_ids":["senseval3.d001.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"responsibility%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018635,"input_uncached_usd":0.002795,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.94736604206264}
{"item_id":"senseval3.d001.s010.t005","gold_sense_keys":["vote%2:41:10::","vote%2:41:11::","vote%2:41:12::","vote%2:41:13::"],"candidates":[{"index":1,"sense_key":"vote%2:41:09::","synset_id":"vote.v.04"},{"index":2,"sense_key":"vote%2:41:10::","synset_id":"vote.v.02"},{"index":3,"sense_key":"vote%2:41:11::","synset_id":"vote.v.03"},{"index":4,"sense_key":"vote%2:41:12::","synset_id":"vote.v.05"},{"index":5,"sense_key":"vote%2:41:13::","synset_id":"vote.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"vote%2:41:13::","call_ids":["senseval3.d001.s010.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"vote%2:41:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":180,"reasoning_output_tokens":164},"cost":{"currency":"USD","total_usd":0.00825,"input_uncached_usd":0.00285,"input_cached_usd":0.0,"output_usd":0.0054,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.006661833031103}
{"item_id":"senseval3.d001.s010.t006","gold_sense_keys":["office%1:26:00::"],"candidates":[{"index":1,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":2,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":3,"sense_key":"office%1:04:02::","synset_id":"position.n.06"},{"index":4,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":5,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":6,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":7,"sense_key":"office%1:04:01::","synset_id":"function.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"office%1:26:00::","call_ids":["senseval3.d001.s010.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"office%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":705,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":103},"cost":{"currency":"USD","total_usd":0.007095000000000001,"input_uncached_usd":0.0035250000000000004,"input_cached_usd":0.0,"output_usd":0.0035700000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.027590333018452}
{"item_id":"senseval3.d001.s010.t007","gold_sense_keys":["deal%2:41:09::"],"candidates":[{"index":1,"sense_key":"deal%2:41:00::","synset_id":"cope.v.01"},{"index":2,"sense_key":"deal%2:40:07::","synset_id":"share.v.04"},{"index":3,"sense_key":"deal%2:32:08::","synset_id":"cover.v.05"},{"index":4,"sense_key":"deal%2:40:00::","synset_id":"deal.v.06"},{"index":5,"sense_key":"deal%2:40:06::","synset_id":"deal.v.12"},{"index":6,"sense_key":"deal%2:41:02::","synset_id":"conduct.v.01"},{"index":7,"sense_key":"deal%2:40:02::","synset_id":"deal.v.09"},{"index":8,"sense_key":"deal%2:41:04::","synset_id":"deal.v.08"},{"index":9,"sense_key":"deal%2:41:09::","synset_id":"deal.v.03"},{"index":10,"sense_key":"deal%2:41:13::","synset_id":"manage.v.02"},{"index":11,"sense_key":"deal%2:31:10::","synset_id":"consider.v.03"},{"index":12,"sense_key":"deal%2:40:11::","synset_id":"deal.v.13"},{"index":13,"sense_key":"deal%2:40:01::","synset_id":"distribute.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"deal%2:41:09::","call_ids":["senseval3.d001.s010.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"deal%2:41:09::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":959,"cached_input_tokens":0,"output_tokens":512,"reasoning_output_tokens":496},"cost":{"currency":"USD","total_usd":0.020155,"input_uncached_usd":0.004795000000000001,"input_cached_usd":0.0,"output_usd":0.01536,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.597569750039838}
{"item_id":"senseval3.d001.s010.t008","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:01:01::","synset_id":"national.a.07"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":5,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":6,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":7,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s010.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":112},"cost":{"currency":"USD","total_usd":0.007050000000000001,"input_uncached_usd":0.0032400000000000003,"input_cached_usd":0.0,"output_usd":0.00381,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.28378429193981}
{"item_id":"senseval3.d001.s010.t009","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":2,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"},{"index":3,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":4,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":5,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":6,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":7,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":8,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":9,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":10,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":11,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"issue%1:09:01::","call_ids":["senseval3.d001.s010.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":924,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":141},"cost":{"currency":"USD","total_usd":0.009330000000000001,"input_uncached_usd":0.00462,"input_cached_usd":0.0,"output_usd":0.00471,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0405971659347415}
{"item_id":"senseval3.d001.s010.t010","gold_sense_keys":["action%1:04:04::"],"candidates":[{"index":1,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":2,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":3,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":4,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":5,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":6,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":7,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":8,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":9,"sense_key":"action%1:04:00::","synset_id":"action.n.10"},{"index":10,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"action%1:04:04::","call_ids":["senseval3.d001.s010.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"action%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":838,"cached_input_tokens":0,"output_tokens":210,"reasoning_output_tokens":194},"cost":{"currency":"USD","total_usd":0.010490000000000001,"input_uncached_usd":0.00419,"input_cached_usd":0.0,"output_usd":0.0063,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.56785679096356}
{"item_id":"senseval3.d001.s010.t011","gold_sense_keys":["cast%2:40:13::"],"candidates":[{"index":1,"sense_key":"cast%2:38:01::","synset_id":"roll.v.12"},{"index":2,"sense_key":"cast%2:36:02::","synset_id":"project.v.10"},{"index":3,"sense_key":"cast%2:35:00::","synset_id":"hurl.v.01"},{"index":4,"sense_key":"cast%2:32:00::","synset_id":"frame.v.04"},{"index":5,"sense_key":"cast%2:29:00::","synset_id":"vomit.v.01"},{"index":6,"sense_key":"cast%2:36:01::","synset_id":"cast.v.03"},{"index":7,"sense_key":"cast%2:35:01::","synset_id":"shed.v.01"},{"index":8,"sense_key":"cast%2:41:14::","synset_id":"cast.v.05"},{"index":9,"sense_key":"cast%2:33:00::","synset_id":"draw.v.14"},{"index":10,"sense_key":"cast%2:40:13::","synset_id":"cast.v.02"},{"index":11,"sense_key":"cast%2:36:00::","synset_id":"cast.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"cast%2:40:13::","call_ids":["senseval3.d001.s010.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"cast%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":886,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.005840000000000001,"input_uncached_usd":0.004430000000000001,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7438602079637349}
{"item_id":"senseval3.d001.s010.t013","gold_sense_keys":["vote%1:04:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":2,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":3,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"},{"index":4,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":5,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vote%1:04:00::","call_ids":["senseval3.d001.s010.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vote%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":288,"reasoning_output_tokens":273},"cost":{"currency":"USD","total_usd":0.011715,"input_uncached_usd":0.003075,"input_cached_usd":0.0,"output_usd":0.00864,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.607210958027281}
{"item_id":"senseval3.d001.s010.t014","gold_sense_keys":["legislation%1:10:00::"],"candidates":[{"index":1,"sense_key":"legislation%1:10:00::","synset_id":"legislation.n.01"},{"index":2,"sense_key":"legislation%1:04:00::","synset_id":"legislation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"legislation%1:10:00::","call_ids":["senseval3.d001.s010.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"legislation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":140},"cost":{"currency":"USD","total_usd":0.006915,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.00468,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.216123500023969}
{"item_id":"senseval3.d001.s010.t015","gold_sense_keys":["impose%2:32:00::"],"candidates":[{"index":1,"sense_key":"impose%2:40:00::","synset_id":"levy.v.01"},{"index":2,"sense_key":"impose%2:32:00::","synset_id":"inflict.v.01"},{"index":3,"sense_key":"impose%2:41:00::","synset_id":"enforce.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"impose%2:32:00::","call_ids":["senseval3.d001.s010.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"impose%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":330,"reasoning_output_tokens":314},"cost":{"currency":"USD","total_usd":0.012379999999999999,"input_uncached_usd":0.00248,"input_cached_usd":0.0,"output_usd":0.0099,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.370328582939692}
{"item_id":"senseval3.d001.s010.t016","gold_sense_keys":["cost%1:07:01::"],"candidates":[{"index":1,"sense_key":"cost%1:07:00::","synset_id":"monetary_value.n.01"},{"index":2,"sense_key":"cost%1:07:01::","synset_id":"price.n.03"},{"index":3,"sense_key":"cost%1:21:00::","synset_id":"cost.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cost%1:07:01::","call_ids":["senseval3.d001.s010.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cost%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":134},"cost":{"currency":"USD","total_usd":0.007165000000000001,"input_uncached_usd":0.0026650000000000003,"input_cached_usd":0.0,"output_usd":0.0045000000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2316190419951454}
{"item_id":"senseval3.d001.s010.t017","gold_sense_keys":["confer%2:40:00::"],"candidates":[{"index":1,"sense_key":"confer%2:32:00::","synset_id":"confer.v.01"},{"index":2,"sense_key":"confer%2:40:00::","synset_id":"confer.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"confer%2:40:00::","call_ids":["senseval3.d001.s010.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"confer%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.003755,"input_uncached_usd":0.002405,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.769963667029515}
{"item_id":"senseval3.d001.s010.t018","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["senseval3.d001.s010.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.005585,"input_uncached_usd":0.0023450000000000003,"input_cached_usd":0.0,"output_usd":0.0032400000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.243013083003461}
{"item_id":"senseval3.d001.s010.t019","gold_sense_keys":["population%1:14:00::"],"candidates":[{"index":1,"sense_key":"population%1:23:00::","synset_id":"population.n.04"},{"index":2,"sense_key":"population%1:09:00::","synset_id":"population.n.03"},{"index":3,"sense_key":"population%1:04:00::","synset_id":"population.n.05"},{"index":4,"sense_key":"population%1:14:00::","synset_id":"population.n.01"},{"index":5,"sense_key":"population%1:14:01::","synset_id":"population.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"population%1:14:00::","call_ids":["senseval3.d001.s010.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"population%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":632,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":79},"cost":{"currency":"USD","total_usd":0.00601,"input_uncached_usd":0.00316,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6906554169254377}
{"item_id":"senseval3.d001.s010.t020","gold_sense_keys":["attend_to%2:31:00::"],"candidates":[{"index":1,"sense_key":"attend_to%2:41:01::","synset_id":"serve.v.10"},{"index":2,"sense_key":"attend_to%2:31:00::","synset_id":"attend_to.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"attend_to%2:31:00::","call_ids":["senseval3.d001.s010.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"attend_to%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.00537,"input_uncached_usd":0.0024000000000000002,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2745623330119997}
{"item_id":"senseval3.d001.s010.t021","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:02::","synset_id":"local.a.03"},{"index":2,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":3,"sense_key":"local%3:01:01::","synset_id":"local.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:00:01::","call_ids":["senseval3.d001.s010.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":396,"reasoning_output_tokens":380},"cost":{"currency":"USD","total_usd":0.01438,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.01188,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.863358833012171}
{"item_id":"senseval3.d001.s010.t022","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":2,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":3,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":4,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":5,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":6,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":7,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"},{"index":8,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":9,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":10,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":11,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"issue%1:09:01::","call_ids":["senseval3.d001.s010.t022__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":924,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02046,"input_uncached_usd":0.00462,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.069596166955307}
{"item_id":"senseval3.d001.s010.t024","gold_sense_keys":["service%1:04:08::"],"candidates":[{"index":1,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":2,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":3,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":4,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":5,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":6,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":7,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":8,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":9,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":10,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":11,"sense_key":"service%1:04:06::","synset_id":"service.n.15"},{"index":12,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":13,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":14,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":15,"sense_key":"service%1:04:08::","synset_id":"service.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"service%1:04:08::","call_ids":["senseval3.d001.s010.t024__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"service%1:04:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":961,"cached_input_tokens":0,"output_tokens":803,"reasoning_output_tokens":781},"cost":{"currency":"USD","total_usd":0.028895,"input_uncached_usd":0.004805,"input_cached_usd":0.0,"output_usd":0.02409,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.573551167035475}
{"item_id":"senseval3.d001.s011.t000","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"},{"index":2,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.004280000000000001,"input_uncached_usd":0.0026300000000000004,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5575317919719964}
{"item_id":"senseval3.d001.s011.t002","gold_sense_keys":["representative%1:18:01::"],"candidates":[{"index":1,"sense_key":"representative%1:18:01::","synset_id":"congressman.n.01"},{"index":2,"sense_key":"representative%1:18:00::","synset_id":"representative.n.01"},{"index":3,"sense_key":"representative%1:09:00::","synset_id":"example.n.01"},{"index":4,"sense_key":"representative%1:18:02::","synset_id":"spokesperson.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"representative%1:18:01::","call_ids":["senseval3.d001.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"representative%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":571,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018695,"input_uncached_usd":0.0028550000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.403260416933335}
{"item_id":"senseval3.d001.s011.t003","gold_sense_keys":["belief%1:09:00::"],"candidates":[{"index":1,"sense_key":"belief%1:09:01::","synset_id":"impression.n.01"},{"index":2,"sense_key":"belief%1:09:00::","synset_id":"belief.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"belief%1:09:00::","call_ids":["senseval3.d001.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"belief%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.00449,"input_uncached_usd":0.00239,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9189238750841469}
{"item_id":"senseval3.d001.s011.t004","gold_sense_keys":["minimalist%5:00:00:moderate:00"],"candidates":[{"index":1,"sense_key":"minimalist%3:01:00::","synset_id":"minimalist.a.01"},{"index":2,"sense_key":"minimalist%5:00:00:moderate:00","synset_id":"minimalist.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"minimalist%5:00:00:moderate:00","call_ids":["senseval3.d001.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"minimalist%5:00:00:moderate:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.003925,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8410421669250354}
{"item_id":"senseval3.d001.s011.t005","gold_sense_keys":["state%1:14:00::","state%1:14:01::"],"candidates":[{"index":1,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":2,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"state%1:14:01::","call_ids":["senseval3.d001.s011.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"state%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":793,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":141},"cost":{"currency":"USD","total_usd":0.008675000000000002,"input_uncached_usd":0.003965000000000001,"input_cached_usd":0.0,"output_usd":0.00471,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.756640417035669}
{"item_id":"senseval3.d001.s011.t006","gold_sense_keys":["willing%3:00:00::"],"candidates":[{"index":1,"sense_key":"willing%3:00:00::","synset_id":"willing.a.01"},{"index":2,"sense_key":"willing%5:00:00:voluntary:01","synset_id":"uncoerced.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"willing%3:00:00::","call_ids":["senseval3.d001.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"willing%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.00438,"input_uncached_usd":0.0024000000000000002,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.998439499991946}
{"item_id":"senseval3.d001.s011.t007","gold_sense_keys":["engage%2:41:06::"],"candidates":[{"index":1,"sense_key":"engage%2:41:06::","synset_id":"prosecute.v.03"},{"index":2,"sense_key":"engage%2:35:00::","synset_id":"engage.v.10"},{"index":3,"sense_key":"engage%2:41:01::","synset_id":"hire.v.01"},{"index":4,"sense_key":"engage%2:31:00::","synset_id":"absorb.v.09"},{"index":5,"sense_key":"engage%2:30:00::","synset_id":"engage.v.06"},{"index":6,"sense_key":"engage%2:40:01::","synset_id":"engage.v.08"},{"index":7,"sense_key":"engage%2:40:00::","synset_id":"lease.v.04"},{"index":8,"sense_key":"engage%2:41:12::","synset_id":"engage.v.07"},{"index":9,"sense_key":"engage%2:41:00::","synset_id":"engage.v.04"},{"index":10,"sense_key":"engage%2:32:00::","synset_id":"betroth.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"engage%2:41:06::","call_ids":["senseval3.d001.s011.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"engage%2:41:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":815,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.005515000000000001,"input_uncached_usd":0.0040750000000000005,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4490634159883484}
{"item_id":"senseval3.d001.s011.t008","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":2,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s011.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018445,"input_uncached_usd":0.002605,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.469299708027393}
{"item_id":"senseval3.d001.s011.t009","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":2,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s011.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":540,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00462,"input_uncached_usd":0.0027,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8414311250671744}
{"item_id":"senseval3.d001.s011.t010","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":5,"sense_key":"member%1:18:00::","synset_id":"member.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"member%1:18:00::","call_ids":["senseval3.d001.s011.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"member%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.004915,"input_uncached_usd":0.003265,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5661533330567181}
{"item_id":"senseval3.d001.s012.t000","gold_sense_keys":["assumption%1:09:00::","assumption%1:10:00::"],"candidates":[{"index":1,"sense_key":"assumption%1:28:00::","synset_id":"assumption.n.04"},{"index":2,"sense_key":"assumption%1:10:00::","synset_id":"premise.n.01"},{"index":3,"sense_key":"assumption%1:04:01::","synset_id":"assumption.n.07"},{"index":4,"sense_key":"assumption%1:04:00::","synset_id":"assumption.n.03"},{"index":5,"sense_key":"assumption%1:11:00::","synset_id":"assumption.n.05"},{"index":6,"sense_key":"assumption%1:07:00::","synset_id":"presumption.n.03"},{"index":7,"sense_key":"assumption%1:09:00::","synset_id":"assumption.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"assumption%1:10:00::","call_ids":["senseval3.d001.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"assumption%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":797,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019825000000000002,"input_uncached_usd":0.003985000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.9209034169325605}
{"item_id":"senseval3.d001.s012.t001","gold_sense_keys":["hold%2:42:02::"],"candidates":[{"index":1,"sense_key":"hold%2:34:00::","synset_id":"carry.v.33"},{"index":2,"sense_key":"hold%2:35:00::","synset_id":"hold.v.02"},{"index":3,"sense_key":"hold%2:42:07::","synset_id":"hold.v.17"},{"index":4,"sense_key":"hold%2:33:00::","synset_id":"hold.v.33"},{"index":5,"sense_key":"hold%2:42:12::","synset_id":"hold.v.26"},{"index":6,"sense_key":"hold%2:42:05::","synset_id":"accommodate.v.04"},{"index":7,"sense_key":"hold%2:35:09::","synset_id":"hold.v.31"},{"index":8,"sense_key":"hold%2:42:08::","synset_id":"defy.v.01"},{"index":9,"sense_key":"hold%2:35:01::","synset_id":"hold.v.10"},{"index":10,"sense_key":"hold%2:38:05::","synset_id":"halt.v.01"},{"index":11,"sense_key":"hold%2:31:04::","synset_id":"hold.v.22"},{"index":12,"sense_key":"hold%2:41:14::","synset_id":"hold.v.29"},{"index":13,"sense_key":"hold%2:42:00::","synset_id":"keep.v.01"},{"index":14,"sense_key":"hold%2:40:02::","synset_id":"retain.v.03"},{"index":15,"sense_key":"hold%2:31:01::","synset_id":"deem.v.01"},{"index":16,"sense_key":"hold%2:42:01::","synset_id":"hold.v.13"},{"index":17,"sense_key":"hold%2:40:00::","synset_id":"have.v.01"},{"index":18,"sense_key":"hold%2:35:03::","synset_id":"restrain.v.03"},{"index":19,"sense_key":"hold%2:37:00::","synset_id":"harbor.v.01"},{"index":20,"sense_key":"hold%2:32:15::","synset_id":"agree.v.01"},{"index":21,"sense_key":"hold%2:31:10::","synset_id":"hold.v.23"},{"index":22,"sense_key":"hold%2:35:02::","synset_id":"hold.v.14"},{"index":23,"sense_key":"hold%2:42:02::","synset_id":"prevail.v.02"},{"index":24,"sense_key":"hold%2:40:04::","synset_id":"bear.v.11"},{"index":25,"sense_key":"hold%2:31:08::","synset_id":"hold.v.16"},{"index":26,"sense_key":"hold%2:33:05::","synset_id":"defend.v.03"},{"index":27,"sense_key":"hold%2:42:13::","synset_id":"hold.v.11"},{"index":28,"sense_key":"hold%2:42:14::","synset_id":"contain.v.05"},{"index":29,"sense_key":"hold%2:41:00::","synset_id":"reserve.v.04"},{"index":30,"sense_key":"hold%2:32:13::","synset_id":"oblige.v.02"},{"index":31,"sense_key":"hold%2:41:15::","synset_id":"control.v.02"},{"index":32,"sense_key":"hold%2:32:11::","synset_id":"declare.v.04"},{"index":33,"sense_key":"hold%2:29:04::","synset_id":"hold.v.36"},{"index":34,"sense_key":"hold%2:36:00::","synset_id":"hold.v.03"},{"index":35,"sense_key":"hold%2:42:04::","synset_id":"apply.v.02"},{"index":36,"sense_key":"hold%2:41:13::","synset_id":"hold.v.28"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":23,"chosen_sense_key":"hold%2:42:02::","call_ids":["senseval3.d001.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":23,"predicted_sense_key":"hold%2:42:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2064,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.01293,"input_uncached_usd":0.010320000000000001,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7489239169517532}
{"item_id":"senseval3.d001.s012.t003","gold_sense_keys":["race%1:11:01::"],"candidates":[{"index":1,"sense_key":"race%1:19:00::","synset_id":"slipstream.n.01"},{"index":2,"sense_key":"race%1:06:00::","synset_id":"raceway.n.01"},{"index":3,"sense_key":"race%1:11:01::","synset_id":"race.n.01"},{"index":4,"sense_key":"race%1:14:01::","synset_id":"subspecies.n.01"},{"index":5,"sense_key":"race%1:11:00::","synset_id":"race.n.02"},{"index":6,"sense_key":"race%1:14:00::","synset_id":"race.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"race%1:11:01::","call_ids":["senseval3.d001.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"race%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":660,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.005520000000000001,"input_uncached_usd":0.0033000000000000004,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7384992910083383}
{"item_id":"senseval3.d001.s012.t004","gold_sense_keys":["face%2:32:00::"],"candidates":[{"index":1,"sense_key":"face%2:35:00::","synset_id":"face.v.08"},{"index":2,"sense_key":"face%2:35:01::","synset_id":"face.v.09"},{"index":3,"sense_key":"face%2:32:00::","synset_id":"confront.v.02"},{"index":4,"sense_key":"face%2:32:03::","synset_id":"confront.v.03"},{"index":5,"sense_key":"face%2:42:02::","synset_id":"face.v.04"},{"index":6,"sense_key":"face%2:42:00::","synset_id":"front.v.01"},{"index":7,"sense_key":"face%2:39:04::","synset_id":"face.v.07"},{"index":8,"sense_key":"face%2:38:00::","synset_id":"face.v.05"},{"index":9,"sense_key":"face%2:33:00::","synset_id":"confront.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"face%2:32:00::","call_ids":["senseval3.d001.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"face%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":821,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.007645000000000001,"input_uncached_usd":0.004105,"input_cached_usd":0.0,"output_usd":0.00354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.0703584579750896}
{"item_id":"senseval3.d001.s012.t007","gold_sense_keys":["have%2:40:00::","have%2:42:12::"],"candidates":[{"index":1,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":2,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":3,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":4,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":6,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":7,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":8,"sense_key":"have%2:35:00::","synset_id":"take.v.35"},{"index":9,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":10,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":11,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":12,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":13,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":14,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":15,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":16,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":17,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":18,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":19,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d001.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1274,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":94},"cost":{"currency":"USD","total_usd":0.009670000000000002,"input_uncached_usd":0.006370000000000001,"input_cached_usd":0.0,"output_usd":0.0033,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.518480416969396}
{"item_id":"senseval3.d001.s012.t008","gold_sense_keys":["incentive%1:16:00::"],"candidates":[{"index":1,"sense_key":"incentive%1:21:00::","synset_id":"bonus.n.02"},{"index":2,"sense_key":"incentive%1:16:00::","synset_id":"incentive.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"incentive%1:16:00::","call_ids":["senseval3.d001.s012.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"incentive%1:16:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":507,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.004305000000000001,"input_uncached_usd":0.0025350000000000004,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5383480419404805}
{"item_id":"senseval3.d001.s012.t010","gold_sense_keys":["lean%2:42:01::"],"candidates":[{"index":1,"sense_key":"lean%2:38:07::","synset_id":"list.v.03"},{"index":2,"sense_key":"lean%2:42:01::","synset_id":"tend.v.01"},{"index":3,"sense_key":"lean%2:31:10::","synset_id":"lean.v.04"},{"index":4,"sense_key":"lean%2:35:00::","synset_id":"lean.v.02"},{"index":5,"sense_key":"lean%2:38:00::","synset_id":"lean.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"lean%2:42:01::","call_ids":["senseval3.d001.s012.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"lean%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00502,"input_uncached_usd":0.00319,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8063855420332402}
{"item_id":"senseval3.d001.s012.t011","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":2,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s012.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":575,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.0047350000000000005,"input_uncached_usd":0.0028750000000000004,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.485285040922463}
{"item_id":"senseval3.d001.s013.t000","gold_sense_keys":["put%2:35:00::"],"candidates":[{"index":1,"sense_key":"put%2:31:00::","synset_id":"arrange.v.07"},{"index":2,"sense_key":"put%2:40:00::","synset_id":"invest.v.01"},{"index":3,"sense_key":"put%2:34:00::","synset_id":"put.v.04"},{"index":4,"sense_key":"put%2:36:13::","synset_id":"put.v.08"},{"index":5,"sense_key":"put%2:35:00::","synset_id":"put.v.01"},{"index":6,"sense_key":"put%2:32:00::","synset_id":"frame.v.04"},{"index":7,"sense_key":"put%2:39:13::","synset_id":"put.v.07"},{"index":8,"sense_key":"put%2:35:05::","synset_id":"put.v.02"},{"index":9,"sense_key":"put%2:31:13::","synset_id":"place.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"put%2:35:00::","call_ids":["senseval3.d001.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"put%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019895,"input_uncached_usd":0.0040550000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.475969541002996}
{"item_id":"senseval3.d001.s013.t001","gold_sense_keys":["office%1:26:00::"],"candidates":[{"index":1,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":2,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":3,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":4,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":5,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":6,"sense_key":"office%1:04:02::","synset_id":"position.n.06"},{"index":7,"sense_key":"office%1:04:00::","synset_id":"office.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"office%1:26:00::","call_ids":["senseval3.d001.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"office%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":760,"cached_input_tokens":0,"output_tokens":169,"reasoning_output_tokens":153},"cost":{"currency":"USD","total_usd":0.008870000000000001,"input_uncached_usd":0.0038000000000000004,"input_cached_usd":0.0,"output_usd":0.00507,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8045247920090333}
{"item_id":"senseval3.d001.s013.t002","gold_sense_keys":["acquire%2:40:00::"],"candidates":[{"index":1,"sense_key":"acquire%2:40:06::","synset_id":"acquire.v.05"},{"index":2,"sense_key":"acquire%2:30:00::","synset_id":"assume.v.03"},{"index":3,"sense_key":"acquire%2:29:00::","synset_id":"grow.v.08"},{"index":4,"sense_key":"acquire%2:30:05::","synset_id":"develop.v.03"},{"index":5,"sense_key":"acquire%2:40:00::","synset_id":"get.v.01"},{"index":6,"sense_key":"acquire%2:42:00::","synset_id":"acquire.v.04"},{"index":7,"sense_key":"acquire%2:31:00::","synset_id":"learn.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"acquire%2:40:00::","call_ids":["senseval3.d001.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"acquire%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":768,"cached_input_tokens":0,"output_tokens":299,"reasoning_output_tokens":283},"cost":{"currency":"USD","total_usd":0.012810000000000002,"input_uncached_usd":0.0038400000000000005,"input_cached_usd":0.0,"output_usd":0.00897,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.981478917063214}
{"item_id":"senseval3.d001.s013.t003","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:00:02::","synset_id":"local.a.03"},{"index":3,"sense_key":"local%3:01:01::","synset_id":"local.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":819,"reasoning_output_tokens":797},"cost":{"currency":"USD","total_usd":0.027350000000000003,"input_uncached_usd":0.0027800000000000004,"input_cached_usd":0.0,"output_usd":0.02457,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.500299832900055}
{"item_id":"senseval3.d001.s013.t004","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["senseval3.d001.s013.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.005715,"input_uncached_usd":0.002625,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1223649999592453}
{"item_id":"senseval3.d001.s013.t007","gold_sense_keys":["prevent%2:41:00::"],"candidates":[{"index":1,"sense_key":"prevent%2:41:01::","synset_id":"prevent.v.01"},{"index":2,"sense_key":"prevent%2:41:00::","synset_id":"prevent.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"prevent%2:41:00::","call_ids":["senseval3.d001.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"prevent%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":93},"cost":{"currency":"USD","total_usd":0.00605,"input_uncached_usd":0.0027800000000000004,"input_cached_usd":0.0,"output_usd":0.00327,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0314035000046715}
{"item_id":"senseval3.d001.s013.t008","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":2,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":3,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":4,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d001.s013.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.004795000000000001,"input_uncached_usd":0.0030550000000000004,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":82.07674345897976}
{"item_id":"senseval3.d001.s013.t010","gold_sense_keys":["bring_home_the_bacon%2:41:00::"],"candidates":[{"index":1,"sense_key":"bring_home_the_bacon%2:40:00::","synset_id":"provide.v.06"},{"index":2,"sense_key":"bring_home_the_bacon%2:41:00::","synset_id":"succeed.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bring_home_the_bacon%2:41:00::","call_ids":["senseval3.d001.s013.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bring_home_the_bacon%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":171,"reasoning_output_tokens":155},"cost":{"currency":"USD","total_usd":0.007945,"input_uncached_usd":0.0028150000000000002,"input_cached_usd":0.0,"output_usd":0.00513,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7221187500981614}
{"item_id":"senseval3.d001.s014.t001","gold_sense_keys":["come_to%2:42:03::"],"candidates":[{"index":1,"sense_key":"come_to%2:41:00::","synset_id":"come_to.v.03"},{"index":2,"sense_key":"come_to%2:29:00::","synset_id":"come_to.v.04"},{"index":3,"sense_key":"come_to%2:39:00::","synset_id":"hit.v.09"},{"index":4,"sense_key":"come_to%2:42:03::","synset_id":"refer.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"come_to%2:42:03::","call_ids":["senseval3.d001.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"come_to%2:42:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.005130000000000001,"input_uncached_usd":0.0031500000000000005,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9864487090380862}
{"item_id":"senseval3.d001.s014.t002","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":3,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":4,"sense_key":"national%3:01:01::","synset_id":"national.a.07"},{"index":5,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":6,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":7,"sense_key":"national%3:00:01::","synset_id":"national.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":717,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":138},"cost":{"currency":"USD","total_usd":0.008205,"input_uncached_usd":0.003585,"input_cached_usd":0.0,"output_usd":0.00462,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.590629416983575}
{"item_id":"senseval3.d001.s014.t003","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"},{"index":3,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval3.d001.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":580,"cached_input_tokens":0,"output_tokens":340,"reasoning_output_tokens":324},"cost":{"currency":"USD","total_usd":0.013099999999999999,"input_uncached_usd":0.0029000000000000002,"input_cached_usd":0.0,"output_usd":0.0102,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.207560083945282}
{"item_id":"senseval3.d001.s014.t004","gold_sense_keys":["making%1:04:00::"],"candidates":[{"index":1,"sense_key":"making%1:07:00::","synset_id":"qualification.n.01"},{"index":2,"sense_key":"making%1:04:00::","synset_id":"devising.n.01"},{"index":3,"sense_key":"making%1:06:00::","synset_id":"making.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"making%1:04:00::","call_ids":["senseval3.d001.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"making%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":600,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00492,"input_uncached_usd":0.003,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5916412500664592}
{"item_id":"senseval3.d001.s015.t000","gold_sense_keys":["race%1:11:01::"],"candidates":[{"index":1,"sense_key":"race%1:11:01::","synset_id":"race.n.01"},{"index":2,"sense_key":"race%1:06:00::","synset_id":"raceway.n.01"},{"index":3,"sense_key":"race%1:14:00::","synset_id":"race.n.03"},{"index":4,"sense_key":"race%1:14:01::","synset_id":"subspecies.n.01"},{"index":5,"sense_key":"race%1:11:00::","synset_id":"race.n.02"},{"index":6,"sense_key":"race%1:19:00::","synset_id":"slipstream.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"race%1:11:01::","call_ids":["senseval3.d001.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"race%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":641,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.005485,"input_uncached_usd":0.0032050000000000004,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.001111791934818}
{"item_id":"senseval3.d001.s015.t002","gold_sense_keys":["incentive%1:16:00::"],"candidates":[{"index":1,"sense_key":"incentive%1:16:00::","synset_id":"incentive.n.01"},{"index":2,"sense_key":"incentive%1:21:00::","synset_id":"bonus.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"incentive%1:16:00::","call_ids":["senseval3.d001.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"incentive%1:16:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.00442,"input_uncached_usd":0.0024400000000000003,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7184838339453563}
{"item_id":"senseval3.d001.s015.t004","gold_sense_keys":["lean%2:42:01::"],"candidates":[{"index":1,"sense_key":"lean%2:38:07::","synset_id":"list.v.03"},{"index":2,"sense_key":"lean%2:42:01::","synset_id":"tend.v.01"},{"index":3,"sense_key":"lean%2:31:10::","synset_id":"lean.v.04"},{"index":4,"sense_key":"lean%2:35:00::","synset_id":"lean.v.02"},{"index":5,"sense_key":"lean%2:38:00::","synset_id":"lean.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"lean%2:42:01::","call_ids":["senseval3.d001.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"lean%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.005015,"input_uncached_usd":0.003095,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.08088545803912}
{"item_id":"senseval3.d001.s016.t000","gold_sense_keys":["president%1:18:04::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":5,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"president%1:18:04::","call_ids":["senseval3.d001.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"president%1:18:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":600,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":86},"cost":{"currency":"USD","total_usd":0.00606,"input_uncached_usd":0.003,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1257744159083813}
{"item_id":"senseval3.d001.s016.t001","gold_sense_keys":["limit%2:30:01::"],"candidates":[{"index":1,"sense_key":"limit%2:30:00::","synset_id":"limit.v.02"},{"index":2,"sense_key":"limit%2:30:01::","synset_id":"restrict.v.03"},{"index":3,"sense_key":"limit%2:32:00::","synset_id":"specify.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"limit%2:30:01::","call_ids":["senseval3.d001.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"limit%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":342,"reasoning_output_tokens":326},"cost":{"currency":"USD","total_usd":0.012830000000000001,"input_uncached_usd":0.0025700000000000002,"input_cached_usd":0.0,"output_usd":0.01026,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.796205832972191}
{"item_id":"senseval3.d001.s016.t002","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:02::","synset_id":"local.a.03"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:01::","synset_id":"local.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":245,"reasoning_output_tokens":229},"cost":{"currency":"USD","total_usd":0.009745,"input_uncached_usd":0.002395,"input_cached_usd":0.0,"output_usd":0.00735,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.487502333940938}
{"item_id":"senseval3.d001.s016.t003","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":2,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"},{"index":3,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"benefit%1:07:00::","call_ids":["senseval3.d001.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":106},"cost":{"currency":"USD","total_usd":0.005900000000000001,"input_uncached_usd":0.0022400000000000002,"input_cached_usd":0.0,"output_usd":0.00366,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7188584160758182}
{"item_id":"senseval3.d001.s016.t005","gold_sense_keys":["particular%5:00:02:specific:00"],"candidates":[{"index":1,"sense_key":"particular%5:00:00:fastidious:00","synset_id":"finical.s.01"},{"index":2,"sense_key":"particular%5:00:00:uncommon:00","synset_id":"especial.s.01"},{"index":3,"sense_key":"particular%5:00:00:specific:00","synset_id":"particular.s.01"},{"index":4,"sense_key":"particular%5:00:02:specific:00","synset_id":"particular.s.02"},{"index":5,"sense_key":"particular%5:00:00:primary:00","synset_id":"particular.s.04"},{"index":6,"sense_key":"particular%5:00:00:careful:00","synset_id":"particular.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"particular%5:00:02:specific:00","call_ids":["senseval3.d001.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"particular%5:00:02:specific:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":124},"cost":{"currency":"USD","total_usd":0.007505000000000001,"input_uncached_usd":0.003305,"input_cached_usd":0.0,"output_usd":0.0042,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8557222080416977}
{"item_id":"senseval3.d001.s016.t008","gold_sense_keys":["president%1:18:04::"],"candidates":[{"index":1,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"},{"index":2,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":3,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":4,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":5,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":6,"sense_key":"president%1:18:02::","synset_id":"president.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"president%1:18:04::","call_ids":["senseval3.d001.s016.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"president%1:18:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":600,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.00546,"input_uncached_usd":0.003,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3740227919770405}
{"item_id":"senseval3.d001.s016.t009","gold_sense_keys":["likely%3:00:00::"],"candidates":[{"index":1,"sense_key":"likely%5:00:00:prospective:00","synset_id":"likely.s.01"},{"index":2,"sense_key":"likely%3:00:04::","synset_id":"probable.a.01"},{"index":3,"sense_key":"likely%5:00:00:credible:00","synset_id":"likely.s.02"},{"index":4,"sense_key":"likely%3:00:00::","synset_id":"likely.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"likely%3:00:00::","call_ids":["senseval3.d001.s016.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"likely%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":227,"reasoning_output_tokens":212},"cost":{"currency":"USD","total_usd":0.00944,"input_uncached_usd":0.0026300000000000004,"input_cached_usd":0.0,"output_usd":0.00681,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.833498666062951}
{"item_id":"senseval3.d001.s016.t010","gold_sense_keys":["effective%3:00:00::","effective%5:00:00:competent:00"],"candidates":[{"index":1,"sense_key":"effective%3:00:00::","synset_id":"effective.a.01"},{"index":2,"sense_key":"effective%5:00:00:operational:00","synset_id":"effective.s.05"},{"index":3,"sense_key":"effective%5:00:00:efficacious:00","synset_id":"effective.s.02"},{"index":4,"sense_key":"effective%5:00:00:actual:00","synset_id":"effective.s.04"},{"index":5,"sense_key":"effective%5:00:00:competent:00","synset_id":"effective.s.01"},{"index":6,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"effective%5:00:00:competent:00","call_ids":["senseval3.d001.s016.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"effective%5:00:00:competent:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019245,"input_uncached_usd":0.0034050000000000005,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.613935291068628}
{"item_id":"senseval3.d001.s016.t011","gold_sense_keys":["prevent%2:41:00::"],"candidates":[{"index":1,"sense_key":"prevent%2:41:00::","synset_id":"prevent.v.02"},{"index":2,"sense_key":"prevent%2:41:01::","synset_id":"prevent.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"prevent%2:41:00::","call_ids":["senseval3.d001.s016.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"prevent%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":99},"cost":{"currency":"USD","total_usd":0.0058200000000000005,"input_uncached_usd":0.0024000000000000002,"input_cached_usd":0.0,"output_usd":0.0034200000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5400045410497114}
{"item_id":"senseval3.d001.s016.t012","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d001.s016.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00459,"input_uncached_usd":0.00267,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.405564291984774}
{"item_id":"senseval3.d001.s016.t016","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:00:02::","synset_id":"local.a.03"},{"index":3,"sense_key":"local%3:01:01::","synset_id":"local.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s016.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":406,"reasoning_output_tokens":390},"cost":{"currency":"USD","total_usd":0.014575,"input_uncached_usd":0.002395,"input_cached_usd":0.0,"output_usd":0.01218,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.292668082984164}
{"item_id":"senseval3.d001.s016.t017","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":2,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"},{"index":3,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"benefit%1:07:00::","call_ids":["senseval3.d001.s016.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":126},"cost":{"currency":"USD","total_usd":0.006495,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.00426,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3938205000013113}
{"item_id":"senseval3.d001.s017.t000","gold_sense_keys":["individual%3:00:00::","individual%5:00:00:independent:00"],"candidates":[{"index":1,"sense_key":"individual%5:00:00:unshared:00","synset_id":"individual.s.02"},{"index":2,"sense_key":"individual%5:00:00:personal:00","synset_id":"individual.s.03"},{"index":3,"sense_key":"individual%5:00:00:independent:00","synset_id":"individual.s.01"},{"index":4,"sense_key":"individual%3:00:00::","synset_id":"individual.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"individual%3:00:00::","call_ids":["senseval3.d001.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"individual%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01847,"input_uncached_usd":0.0026300000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.5259147910401225}
{"item_id":"senseval3.d001.s017.t002","gold_sense_keys":["standing%1:26:00::"],"candidates":[{"index":1,"sense_key":"standing%1:10:00::","synset_id":"standing.n.02"},{"index":2,"sense_key":"standing%1:26:00::","synset_id":"standing.n.01"},{"index":3,"sense_key":"standing%1:04:00::","synset_id":"standing.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"standing%1:26:00::","call_ids":["senseval3.d001.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"standing%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.004765,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.177539833006449}
{"item_id":"senseval3.d001.s017.t003","gold_sense_keys":["enhance%2:30:01::"],"candidates":[{"index":1,"sense_key":"enhance%2:30:00::","synset_id":"enhance.v.01"},{"index":2,"sense_key":"enhance%2:30:01::","synset_id":"enhance.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"enhance%2:30:01::","call_ids":["senseval3.d001.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"enhance%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":328,"reasoning_output_tokens":313},"cost":{"currency":"USD","total_usd":0.011910000000000002,"input_uncached_usd":0.0020700000000000002,"input_cached_usd":0.0,"output_usd":0.00984,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.512989458045922}
{"item_id":"senseval3.d001.s018.t001","gold_sense_keys":["simple%3:00:02::","simple%5:00:00:easy:01"],"candidates":[{"index":1,"sense_key":"simple%5:00:00:easy:01","synset_id":"elementary.s.01"},{"index":2,"sense_key":"simple%5:00:00:retarded:00","synset_id":"dim-witted.s.01"},{"index":3,"sense_key":"simple%3:00:01::","synset_id":"simple.a.06"},{"index":4,"sense_key":"simple%5:00:00:naive:00","synset_id":"childlike.s.02"},{"index":5,"sense_key":"simple%5:00:02:plain:01","synset_id":"bare.s.04"},{"index":6,"sense_key":"simple%3:00:02::","synset_id":"simple.a.01"},{"index":7,"sense_key":"simple%5:00:00:plain:01","synset_id":"simple.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"simple%3:00:02::","call_ids":["senseval3.d001.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"simple%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019200000000000002,"input_uncached_usd":0.00336,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.922876833938062}
{"item_id":"senseval3.d001.s018.t002","gold_sense_keys":["appear%2:39:00::"],"candidates":[{"index":1,"sense_key":"appear%2:30:02::","synset_id":"appear.v.05"},{"index":2,"sense_key":"appear%2:39:00::","synset_id":"look.v.02"},{"index":3,"sense_key":"appear%2:30:01::","synset_id":"appear.v.03"},{"index":4,"sense_key":"appear%2:41:03::","synset_id":"appear.v.07"},{"index":5,"sense_key":"appear%2:30:00::","synset_id":"appear.v.02"},{"index":6,"sense_key":"appear%2:36:00::","synset_id":"appear.v.06"},{"index":7,"sense_key":"appear%2:39:01::","synset_id":"appear.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"appear%2:39:00::","call_ids":["senseval3.d001.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"appear%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":714,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01941,"input_uncached_usd":0.0035700000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.103649917058647}
{"item_id":"senseval3.d001.s018.t003","gold_sense_keys":["explain%2:32:02::"],"candidates":[{"index":1,"sense_key":"explain%2:32:00::","synset_id":"explain.v.01"},{"index":2,"sense_key":"explain%2:32:02::","synset_id":"excuse.v.03"},{"index":3,"sense_key":"explain%2:32:01::","synset_id":"explain.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"explain%2:32:02::","call_ids":["senseval3.d001.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"explain%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018355,"input_uncached_usd":0.0025150000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.36911249998957}
{"item_id":"senseval3.d001.s018.t004","gold_sense_keys":["several%5:00:00:some:00"],"candidates":[{"index":1,"sense_key":"several%5:00:00:some:00","synset_id":"several.s.01"},{"index":2,"sense_key":"several%5:00:00:individual:00","synset_id":"respective.s.01"},{"index":3,"sense_key":"several%5:00:00:different:00","synset_id":"several.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"several%5:00:00:some:00","call_ids":["senseval3.d001.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"several%5:00:00:some:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.00451,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7317988750291988}
{"item_id":"senseval3.d001.s018.t005","gold_sense_keys":["thing%1:09:02::"],"candidates":[{"index":1,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":2,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":3,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":6,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":7,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":8,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":11,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":12,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"thing%1:09:02::","call_ids":["senseval3.d001.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"thing%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":764,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01966,"input_uncached_usd":0.0038200000000000005,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.196177375037223}
{"item_id":"senseval3.d001.s019.t000","gold_sense_keys":["increase%2:30:00::"],"candidates":[{"index":1,"sense_key":"increase%2:30:02::","synset_id":"increase.v.02"},{"index":2,"sense_key":"increase%2:30:00::","synset_id":"increase.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"increase%2:30:00::","call_ids":["senseval3.d001.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"increase%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.003985000000000001,"input_uncached_usd":0.002245,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6536215000087395}
{"item_id":"senseval3.d001.s019.t001","gold_sense_keys":["take%2:30:01::"],"candidates":[{"index":1,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":2,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":3,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":4,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":5,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":6,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":7,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":8,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":9,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":10,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":11,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":12,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":13,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":14,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"},{"index":15,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":16,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":17,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":18,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":19,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":20,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":21,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":22,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":23,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":24,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":25,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":26,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":27,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":28,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":29,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":30,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":31,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":32,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":33,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":34,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":35,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":36,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":37,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":38,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":39,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":40,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":41,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":42,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":20,"chosen_sense_key":"take%2:30:01::","call_ids":["senseval3.d001.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":20,"predicted_sense_key":"take%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2239,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":104},"cost":{"currency":"USD","total_usd":0.014794999999999999,"input_uncached_usd":0.011195,"input_cached_usd":0.0,"output_usd":0.0036,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6236777500016615}
{"item_id":"senseval3.d001.s019.t004","gold_sense_keys":["past%3:00:00::"],"candidates":[{"index":1,"sense_key":"past%3:00:00::","synset_id":"past.a.01"},{"index":2,"sense_key":"past%5:00:00:outgoing:00","synset_id":"past.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"past%3:00:00::","call_ids":["senseval3.d001.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"past%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.004405,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.267561958055012}
{"item_id":"senseval3.d001.s019.t006","gold_sense_keys":["election%1:26:00::"],"candidates":[{"index":1,"sense_key":"election%1:26:00::","synset_id":"election.n.03"},{"index":2,"sense_key":"election%1:04:00::","synset_id":"election.n.02"},{"index":3,"sense_key":"election%1:04:01::","synset_id":"election.n.01"},{"index":4,"sense_key":"election%1:09:00::","synset_id":"election.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"election%1:26:00::","call_ids":["senseval3.d001.s019.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"election%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":523,"cached_input_tokens":0,"output_tokens":329,"reasoning_output_tokens":313},"cost":{"currency":"USD","total_usd":0.012485,"input_uncached_usd":0.002615,"input_cached_usd":0.0,"output_usd":0.00987,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.195418540970422}
{"item_id":"senseval3.d001.s019.t007","gold_sense_keys":["president%1:18:04::"],"candidates":[{"index":1,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":2,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":3,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":4,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"},{"index":5,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":6,"sense_key":"president%1:18:00::","synset_id":"president.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"president%1:04:04::","call_ids":["senseval3.d001.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"president%1:04:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01892,"input_uncached_usd":0.0030800000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.266803791979328}
{"item_id":"senseval3.d001.s019.t008","gold_sense_keys":["advent%1:04:00::"],"candidates":[{"index":1,"sense_key":"advent%1:11:00::","synset_id":"second_coming.n.01"},{"index":2,"sense_key":"advent%1:04:00::","synset_id":"advent.n.01"},{"index":3,"sense_key":"advent%1:28:01::","synset_id":"advent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"advent%1:04:00::","call_ids":["senseval3.d001.s019.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"advent%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.00421,"input_uncached_usd":0.0024700000000000004,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4001812089700252}
{"item_id":"senseval3.d001.s019.t009","gold_sense_keys":["government%1:14:00::"],"candidates":[{"index":1,"sense_key":"government%1:09:00::","synset_id":"government.n.03"},{"index":2,"sense_key":"government%1:09:01::","synset_id":"politics.n.02"},{"index":3,"sense_key":"government%1:14:00::","synset_id":"government.n.01"},{"index":4,"sense_key":"government%1:04:00::","synset_id":"government.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"government%1:14:00::","call_ids":["senseval3.d001.s019.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"government%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":223,"reasoning_output_tokens":207},"cost":{"currency":"USD","total_usd":0.009475,"input_uncached_usd":0.002785,"input_cached_usd":0.0,"output_usd":0.00669,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.623745959019288}
{"item_id":"senseval3.d001.s019.t010","gold_sense_keys":["occupy%2:41:04::"],"candidates":[{"index":1,"sense_key":"occupy%2:40:08::","synset_id":"take.v.02"},{"index":2,"sense_key":"occupy%2:41:04::","synset_id":"fill.v.04"},{"index":3,"sense_key":"occupy%2:42:04::","synset_id":"concern.v.02"},{"index":4,"sense_key":"occupy%2:33:00::","synset_id":"invade.v.01"},{"index":5,"sense_key":"occupy%2:42:01::","synset_id":"occupy.v.03"},{"index":6,"sense_key":"occupy%2:31:00::","synset_id":"absorb.v.09"},{"index":7,"sense_key":"occupy%2:41:00::","synset_id":"busy.v.01"},{"index":8,"sense_key":"occupy%2:42:00::","synset_id":"occupy.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"occupy%2:41:04::","call_ids":["senseval3.d001.s019.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"occupy%2:41:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":739,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":126},"cost":{"currency":"USD","total_usd":0.007955,"input_uncached_usd":0.0036950000000000004,"input_cached_usd":0.0,"output_usd":0.00426,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.057650874950923}
{"item_id":"senseval3.d001.s019.t011","gold_sense_keys":["role%1:04:00::"],"candidates":[{"index":1,"sense_key":"role%1:04:00::","synset_id":"function.n.03"},{"index":2,"sense_key":"role%1:09:00::","synset_id":"character.n.04"},{"index":3,"sense_key":"role%1:07:00::","synset_id":"function.n.02"},{"index":4,"sense_key":"role%1:04:01::","synset_id":"role.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"role%1:04:00::","call_ids":["senseval3.d001.s019.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"role%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":387,"reasoning_output_tokens":371},"cost":{"currency":"USD","total_usd":0.014450000000000001,"input_uncached_usd":0.00284,"input_cached_usd":0.0,"output_usd":0.01161,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.137417833087966}
{"item_id":"senseval3.d001.s019.t012","gold_sense_keys":["society%1:14:00::"],"candidates":[{"index":1,"sense_key":"society%1:26:00::","synset_id":"company.n.03"},{"index":2,"sense_key":"society%1:14:00::","synset_id":"society.n.01"},{"index":3,"sense_key":"society%1:14:01::","synset_id":"club.n.02"},{"index":4,"sense_key":"society%1:14:02::","synset_id":"society.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"society%1:14:00::","call_ids":["senseval3.d001.s019.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"society%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.004935,"input_uncached_usd":0.0027150000000000004,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.655819582985714}
{"item_id":"senseval3.d001.s019.t013","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:26:00::","call_ids":["senseval3.d001.s019.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":388,"reasoning_output_tokens":372},"cost":{"currency":"USD","total_usd":0.014124999999999999,"input_uncached_usd":0.0024850000000000002,"input_cached_usd":0.0,"output_usd":0.011640000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.7121678330004215}
{"item_id":"senseval3.d001.s019.t016","gold_sense_keys":["race%1:11:01::"],"candidates":[{"index":1,"sense_key":"race%1:19:00::","synset_id":"slipstream.n.01"},{"index":2,"sense_key":"race%1:06:00::","synset_id":"raceway.n.01"},{"index":3,"sense_key":"race%1:14:00::","synset_id":"race.n.03"},{"index":4,"sense_key":"race%1:14:01::","synset_id":"subspecies.n.01"},{"index":5,"sense_key":"race%1:11:01::","synset_id":"race.n.01"},{"index":6,"sense_key":"race%1:11:00::","synset_id":"race.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"race%1:11:01::","call_ids":["senseval3.d001.s019.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"race%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.005,"input_uncached_usd":0.00299,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1509439169894904}
{"item_id":"senseval3.d001.s020.t000","gold_sense_keys":["explain%2:32:02::"],"candidates":[{"index":1,"sense_key":"explain%2:32:02::","synset_id":"excuse.v.03"},{"index":2,"sense_key":"explain%2:32:01::","synset_id":"explain.v.02"},{"index":3,"sense_key":"explain%2:32:00::","synset_id":"explain.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"explain%2:32:00::","call_ids":["senseval3.d001.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"explain%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":2424,"reasoning_output_tokens":2384},"cost":{"currency":"USD","total_usd":0.07530500000000001,"input_uncached_usd":0.002585,"input_cached_usd":0.0,"output_usd":0.07272,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":40.178617083933204}
{"item_id":"senseval3.d001.s020.t002","gold_sense_keys":["hold%2:31:01::"],"candidates":[{"index":1,"sense_key":"hold%2:33:00::","synset_id":"hold.v.33"},{"index":2,"sense_key":"hold%2:42:14::","synset_id":"contain.v.05"},{"index":3,"sense_key":"hold%2:41:14::","synset_id":"hold.v.29"},{"index":4,"sense_key":"hold%2:31:01::","synset_id":"deem.v.01"},{"index":5,"sense_key":"hold%2:35:09::","synset_id":"hold.v.31"},{"index":6,"sense_key":"hold%2:36:00::","synset_id":"hold.v.03"},{"index":7,"sense_key":"hold%2:32:15::","synset_id":"agree.v.01"},{"index":8,"sense_key":"hold%2:42:01::","synset_id":"hold.v.13"},{"index":9,"sense_key":"hold%2:40:02::","synset_id":"retain.v.03"},{"index":10,"sense_key":"hold%2:40:04::","synset_id":"bear.v.11"},{"index":11,"sense_key":"hold%2:29:04::","synset_id":"hold.v.36"},{"index":12,"sense_key":"hold%2:32:13::","synset_id":"oblige.v.02"},{"index":13,"sense_key":"hold%2:38:05::","synset_id":"halt.v.01"},{"index":14,"sense_key":"hold%2:42:08::","synset_id":"defy.v.01"},{"index":15,"sense_key":"hold%2:32:11::","synset_id":"declare.v.04"},{"index":16,"sense_key":"hold%2:33:05::","synset_id":"defend.v.03"},{"index":17,"sense_key":"hold%2:41:13::","synset_id":"hold.v.28"},{"index":18,"sense_key":"hold%2:35:03::","synset_id":"restrain.v.03"},{"index":19,"sense_key":"hold%2:35:00::","synset_id":"hold.v.02"},{"index":20,"sense_key":"hold%2:35:02::","synset_id":"hold.v.14"},{"index":21,"sense_key":"hold%2:37:00::","synset_id":"harbor.v.01"},{"index":22,"sense_key":"hold%2:31:04::","synset_id":"hold.v.22"},{"index":23,"sense_key":"hold%2:31:08::","synset_id":"hold.v.16"},{"index":24,"sense_key":"hold%2:34:00::","synset_id":"carry.v.33"},{"index":25,"sense_key":"hold%2:42:00::","synset_id":"keep.v.01"},{"index":26,"sense_key":"hold%2:41:15::","synset_id":"control.v.02"},{"index":27,"sense_key":"hold%2:42:05::","synset_id":"accommodate.v.04"},{"index":28,"sense_key":"hold%2:42:12::","synset_id":"hold.v.26"},{"index":29,"sense_key":"hold%2:42:04::","synset_id":"apply.v.02"},{"index":30,"sense_key":"hold%2:42:02::","synset_id":"prevail.v.02"},{"index":31,"sense_key":"hold%2:42:13::","synset_id":"hold.v.11"},{"index":32,"sense_key":"hold%2:31:10::","synset_id":"hold.v.23"},{"index":33,"sense_key":"hold%2:41:00::","synset_id":"reserve.v.04"},{"index":34,"sense_key":"hold%2:40:00::","synset_id":"have.v.01"},{"index":35,"sense_key":"hold%2:35:01::","synset_id":"hold.v.10"},{"index":36,"sense_key":"hold%2:42:07::","synset_id":"hold.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"hold%2:31:01::","call_ids":["senseval3.d001.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"hold%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2009,"cached_input_tokens":0,"output_tokens":237,"reasoning_output_tokens":221},"cost":{"currency":"USD","total_usd":0.017155,"input_uncached_usd":0.010045,"input_cached_usd":0.0,"output_usd":0.00711,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.409583917004056}
{"item_id":"senseval3.d001.s020.t003","gold_sense_keys":["disdain%1:12:00::"],"candidates":[{"index":1,"sense_key":"disdain%1:12:00::","synset_id":"contempt.n.01"},{"index":2,"sense_key":"disdain%1:10:00::","synset_id":"condescension.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"disdain%1:12:00::","call_ids":["senseval3.d001.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"disdain%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.004635,"input_uncached_usd":0.0023850000000000004,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9661094590555876}
{"item_id":"senseval3.d001.s020.t004","gold_sense_keys":["love%2:37:00::"],"candidates":[{"index":1,"sense_key":"love%2:37:02::","synset_id":"love.v.02"},{"index":2,"sense_key":"love%2:37:01::","synset_id":"love.v.03"},{"index":3,"sense_key":"love%2:37:00::","synset_id":"love.v.01"},{"index":4,"sense_key":"love%2:35:00::","synset_id":"sleep_together.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"love%2:37:00::","call_ids":["senseval3.d001.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"love%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":542,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.00487,"input_uncached_usd":0.00271,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.80381395900622}
{"item_id":"senseval3.d001.s020.t007","gold_sense_keys":["representative%1:18:01::"],"candidates":[{"index":1,"sense_key":"representative%1:18:02::","synset_id":"spokesperson.n.01"},{"index":2,"sense_key":"representative%1:18:01::","synset_id":"congressman.n.01"},{"index":3,"sense_key":"representative%1:18:00::","synset_id":"representative.n.01"},{"index":4,"sense_key":"representative%1:09:00::","synset_id":"example.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"representative%1:18:01::","call_ids":["senseval3.d001.s020.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"representative%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":395,"reasoning_output_tokens":380},"cost":{"currency":"USD","total_usd":0.014605000000000002,"input_uncached_usd":0.002755,"input_cached_usd":0.0,"output_usd":0.011850000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.8588628330035135}
{"item_id":"senseval3.d001.s020.t008","gold_sense_keys":["individual%3:00:00::","individual%5:00:00:independent:00"],"candidates":[{"index":1,"sense_key":"individual%5:00:00:personal:00","synset_id":"individual.s.03"},{"index":2,"sense_key":"individual%3:00:00::","synset_id":"individual.a.01"},{"index":3,"sense_key":"individual%5:00:00:unshared:00","synset_id":"individual.s.02"},{"index":4,"sense_key":"individual%5:00:00:independent:00","synset_id":"individual.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"individual%3:00:00::","call_ids":["senseval3.d001.s020.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"individual%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018685,"input_uncached_usd":0.0028450000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.290603791945614}
{"item_id":"senseval3.d001.s020.t010","gold_sense_keys":["constituent%1:18:00::"],"candidates":[{"index":1,"sense_key":"constituent%1:18:00::","synset_id":"constituent.n.02"},{"index":2,"sense_key":"constituent%1:06:00::","synset_id":"component.n.03"},{"index":3,"sense_key":"constituent%1:10:00::","synset_id":"constituent.n.04"},{"index":4,"sense_key":"constituent%1:09:00::","synset_id":"component.n.01"},{"index":5,"sense_key":"constituent%1:24:00::","synset_id":"part.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"constituent%1:18:00::","call_ids":["senseval3.d001.s020.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"constituent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.0049050000000000005,"input_uncached_usd":0.0033450000000000003,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7612074579810724}
{"item_id":"senseval3.d001.s020.t011","gold_sense_keys":["appreciate%2:37:00::"],"candidates":[{"index":1,"sense_key":"appreciate%2:37:00::","synset_id":"appreciate.v.01"},{"index":2,"sense_key":"appreciate%2:30:00::","synset_id":"appreciate.v.04"},{"index":3,"sense_key":"appreciate%2:40:00::","synset_id":"prize.v.01"},{"index":4,"sense_key":"appreciate%2:31:00::","synset_id":"appreciate.v.02"},{"index":5,"sense_key":"appreciate%2:30:01::","synset_id":"appreciate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"appreciate%2:31:00::","call_ids":["senseval3.d001.s020.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"appreciate%2:31:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":576,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01872,"input_uncached_usd":0.00288,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.302799582947046}
{"item_id":"senseval3.d001.s020.t012","gold_sense_keys":["specific%3:00:00::"],"candidates":[{"index":1,"sense_key":"specific%3:00:00::","synset_id":"specific.a.01"},{"index":2,"sense_key":"specific%3:00:02::","synset_id":"specific.a.04"},{"index":3,"sense_key":"specific%3:01:00::","synset_id":"specific.a.03"},{"index":4,"sense_key":"specific%5:00:00:specified:00","synset_id":"specific.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"specific%3:00:00::","call_ids":["senseval3.d001.s020.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"specific%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":554,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":79},"cost":{"currency":"USD","total_usd":0.00559,"input_uncached_usd":0.0027700000000000003,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.392727458034642}
{"item_id":"senseval3.d001.s020.t013","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["senseval3.d001.s020.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.004750000000000001,"input_uncached_usd":0.00235,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9890377919655293}
{"item_id":"senseval3.d001.s020.t015","gold_sense_keys":["win%2:40:00::"],"candidates":[{"index":1,"sense_key":"win%2:40:00::","synset_id":"acquire.v.05"},{"index":2,"sense_key":"win%2:33:01::","synset_id":"gain.v.05"},{"index":3,"sense_key":"win%2:41:04::","synset_id":"succeed.v.01"},{"index":4,"sense_key":"win%2:33:00::","synset_id":"win.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"win%2:40:00::","call_ids":["senseval3.d001.s020.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"win%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.006525,"input_uncached_usd":0.0030450000000000004,"input_cached_usd":0.0,"output_usd":0.00348,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8010942499386147}
{"item_id":"senseval3.d001.s020.t016","gold_sense_keys":["overall%5:00:00:gross:00"],"candidates":[{"index":1,"sense_key":"overall%5:00:00:general:00","synset_id":"overall.s.01"},{"index":2,"sense_key":"overall%5:00:00:gross:00","synset_id":"overall.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"overall%5:00:00:gross:00","call_ids":["senseval3.d001.s020.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"overall%5:00:00:gross:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.004525,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.882687250035815}
{"item_id":"senseval3.d001.s020.t017","gold_sense_keys":["cost%1:07:01::"],"candidates":[{"index":1,"sense_key":"cost%1:21:00::","synset_id":"cost.n.01"},{"index":2,"sense_key":"cost%1:07:01::","synset_id":"price.n.03"},{"index":3,"sense_key":"cost%1:07:00::","synset_id":"monetary_value.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cost%1:21:00::","call_ids":["senseval3.d001.s020.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cost%1:21:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018505,"input_uncached_usd":0.0026650000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.497950791963376}
{"item_id":"senseval3.d001.s020.t019","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":2,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":3,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":4,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d001.s020.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.00485,"input_uncached_usd":0.0027800000000000004,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2198895829496905}
{"item_id":"senseval3.d001.s020.t021","gold_sense_keys":["do%2:29:09::"],"candidates":[{"index":1,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":2,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":3,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":4,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":5,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":6,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":7,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":8,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":9,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":10,"sense_key":"do%2:38:00::","synset_id":"do.v.13"},{"index":11,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":12,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":13,"sense_key":"do%2:41:01::","synset_id":"make.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"do%2:36:01::","call_ids":["senseval3.d001.s020.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"do%2:36:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":979,"cached_input_tokens":0,"output_tokens":482,"reasoning_output_tokens":466},"cost":{"currency":"USD","total_usd":0.019355,"input_uncached_usd":0.004895,"input_cached_usd":0.0,"output_usd":0.01446,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.908522375044413}
{"item_id":"senseval3.d001.s021.t001","gold_sense_keys":["suggest%2:32:02::"],"candidates":[{"index":1,"sense_key":"suggest%2:32:04::","synset_id":"hint.v.01"},{"index":2,"sense_key":"suggest%2:32:01::","synset_id":"suggest.v.05"},{"index":3,"sense_key":"suggest%2:32:00::","synset_id":"propose.v.01"},{"index":4,"sense_key":"suggest%2:32:03::","synset_id":"indicate.v.05"},{"index":5,"sense_key":"suggest%2:32:02::","synset_id":"suggest.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"suggest%2:32:02::","call_ids":["senseval3.d001.s021.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"suggest%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":90},"cost":{"currency":"USD","total_usd":0.006070000000000001,"input_uncached_usd":0.00289,"input_cached_usd":0.0,"output_usd":0.00318,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.351132499985397}
{"item_id":"senseval3.d001.s021.t004","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":2,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":6,"sense_key":"national%3:01:01::","synset_id":"national.a.07"},{"index":7,"sense_key":"national%3:01:00::","synset_id":"national.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s021.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":655,"cached_input_tokens":0,"output_tokens":174,"reasoning_output_tokens":158},"cost":{"currency":"USD","total_usd":0.008495,"input_uncached_usd":0.003275,"input_cached_usd":0.0,"output_usd":0.00522,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7835674999514595}
{"item_id":"senseval3.d001.s021.t005","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":2,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval3.d001.s021.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":328,"reasoning_output_tokens":313},"cost":{"currency":"USD","total_usd":0.012430000000000002,"input_uncached_usd":0.0025900000000000003,"input_cached_usd":0.0,"output_usd":0.00984,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.538606374990195}
{"item_id":"senseval3.d001.s021.t006","gold_sense_keys":["making%1:04:00::"],"candidates":[{"index":1,"sense_key":"making%1:07:00::","synset_id":"qualification.n.01"},{"index":2,"sense_key":"making%1:06:00::","synset_id":"making.n.03"},{"index":3,"sense_key":"making%1:04:00::","synset_id":"devising.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"making%1:04:00::","call_ids":["senseval3.d001.s021.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"making%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":539,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.004915000000000001,"input_uncached_usd":0.0026950000000000003,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.63023929100018}
{"item_id":"senseval3.d001.s021.t007","gold_sense_keys":["relative%5:00:00:proportionate:00"],"candidates":[{"index":1,"sense_key":"relative%5:00:00:proportionate:00","synset_id":"proportional.s.01"},{"index":2,"sense_key":"relative%3:00:00::","synset_id":"relative.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"relative%5:00:00:proportionate:00","call_ids":["senseval3.d001.s021.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"relative%5:00:00:proportionate:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":169,"reasoning_output_tokens":153},"cost":{"currency":"USD","total_usd":0.0075000000000000015,"input_uncached_usd":0.0024300000000000003,"input_cached_usd":0.0,"output_usd":0.00507,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.36057462496683}
{"item_id":"senseval3.d001.s021.t008","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":2,"sense_key":"local%3:00:02::","synset_id":"local.a.03"},{"index":3,"sense_key":"local%3:00:01::","synset_id":"local.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s021.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":507,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018375,"input_uncached_usd":0.0025350000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.639552416978404}
{"item_id":"senseval3.d001.s021.t009","gold_sense_keys":["have%2:30:01::","have%2:40:00::","have%2:42:00::","have%2:42:12::"],"candidates":[{"index":1,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":2,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":3,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":4,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":5,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":6,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":7,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":8,"sense_key":"have%2:35:00::","synset_id":"take.v.35"},{"index":9,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":10,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":11,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":12,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":13,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":14,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":15,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":16,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":17,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":18,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":19,"sense_key":"have%2:42:12::","synset_id":"have.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d001.s021.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1224,"cached_input_tokens":0,"output_tokens":404,"reasoning_output_tokens":389},"cost":{"currency":"USD","total_usd":0.01824,"input_uncached_usd":0.0061200000000000004,"input_cached_usd":0.0,"output_usd":0.01212,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.9791525839827955}
{"item_id":"senseval3.d001.s021.t011","gold_sense_keys":["office%1:26:00::"],"candidates":[{"index":1,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":2,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":3,"sense_key":"office%1:04:02::","synset_id":"position.n.06"},{"index":4,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":5,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":6,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":7,"sense_key":"office%1:14:00::","synset_id":"office.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"office%1:26:00::","call_ids":["senseval3.d001.s021.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"office%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.0067350000000000005,"input_uncached_usd":0.0035550000000000004,"input_cached_usd":0.0,"output_usd":0.00318,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5665505409706384}
{"item_id":"senseval3.d001.s022.t002","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"},{"index":2,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":3,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":4,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":5,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"member%1:18:00::","call_ids":["senseval3.d001.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"member%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.005525,"input_uncached_usd":0.0031850000000000003,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.677358250017278}
{"item_id":"senseval3.d001.s022.t003","gold_sense_keys":["vulnerable%3:00:00::"],"candidates":[{"index":1,"sense_key":"vulnerable%3:00:00::","synset_id":"vulnerable.a.01"},{"index":2,"sense_key":"vulnerable%5:00:00:susceptible:00","synset_id":"vulnerable.s.01"},{"index":3,"sense_key":"vulnerable%5:00:00:weak:00","synset_id":"vulnerable.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vulnerable%3:00:00::","call_ids":["senseval3.d001.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vulnerable%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":203,"reasoning_output_tokens":187},"cost":{"currency":"USD","total_usd":0.00859,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.00609,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.572197083034553}
{"item_id":"senseval3.d001.s022.t005","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":3,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"become%2:30:00::","call_ids":["senseval3.d001.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"become%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":320,"reasoning_output_tokens":304},"cost":{"currency":"USD","total_usd":0.012355,"input_uncached_usd":0.002755,"input_cached_usd":0.0,"output_usd":0.009600000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.493953041965142}
{"item_id":"senseval3.d001.s022.t006","gold_sense_keys":["immune%5:00:00:unaffected:00"],"candidates":[{"index":1,"sense_key":"immune%5:00:00:unaffected:00","synset_id":"immune.s.03"},{"index":2,"sense_key":"immune%3:01:00::","synset_id":"immune.a.01"},{"index":3,"sense_key":"immune%5:00:00:exempt:00","synset_id":"immune.s.01"},{"index":4,"sense_key":"immune%5:00:00:unsusceptible:00","synset_id":"immune.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"immune%5:00:00:exempt:00","call_ids":["senseval3.d001.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"immune%5:00:00:exempt:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":319,"reasoning_output_tokens":304},"cost":{"currency":"USD","total_usd":0.012255,"input_uncached_usd":0.0026850000000000003,"input_cached_usd":0.0,"output_usd":0.00957,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.388755250023678}
{"item_id":"senseval3.d001.s022.t007","gold_sense_keys":["defeat%1:11:00::"],"candidates":[{"index":1,"sense_key":"defeat%1:11:00::","synset_id":"defeat.n.01"},{"index":2,"sense_key":"defeat%1:12:00::","synset_id":"frustration.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"defeat%1:11:00::","call_ids":["senseval3.d001.s022.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"defeat%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.004445,"input_uncached_usd":0.0023450000000000003,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9982697078958154}
{"item_id":"senseval3.d001.s023.t000","gold_sense_keys":["exception%1:09:01::"],"candidates":[{"index":1,"sense_key":"exception%1:10:00::","synset_id":"exception.n.03"},{"index":2,"sense_key":"exception%1:09:01::","synset_id":"exception.n.02"},{"index":3,"sense_key":"exception%1:09:00::","synset_id":"exception.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"exception%1:09:01::","call_ids":["senseval3.d001.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"exception%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.004370000000000001,"input_uncached_usd":0.0026300000000000004,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9281259160488844}
{"item_id":"senseval3.d001.s023.t002","gold_sense_keys":["trend%1:15:02::"],"candidates":[{"index":1,"sense_key":"trend%1:09:00::","synset_id":"drift.n.05"},{"index":2,"sense_key":"trend%1:09:02::","synset_id":"vogue.n.01"},{"index":3,"sense_key":"trend%1:15:02::","synset_id":"tendency.n.04"},{"index":4,"sense_key":"trend%1:15:01::","synset_id":"course.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trend%1:09:00::","call_ids":["senseval3.d001.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trend%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":590,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01879,"input_uncached_usd":0.0029500000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.615775082958862}
{"item_id":"senseval3.d001.s023.t003","gold_sense_keys":["defeat%1:11:00::"],"candidates":[{"index":1,"sense_key":"defeat%1:12:00::","synset_id":"frustration.n.01"},{"index":2,"sense_key":"defeat%1:11:00::","synset_id":"defeat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"defeat%1:11:00::","call_ids":["senseval3.d001.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"defeat%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.004710000000000001,"input_uncached_usd":0.0024300000000000003,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8446206250227988}
{"item_id":"senseval3.d001.s023.t004","gold_sense_keys":["freshman%1:18:01::"],"candidates":[{"index":1,"sense_key":"freshman%1:18:00::","synset_id":"freshman.n.01"},{"index":2,"sense_key":"freshman%1:18:01::","synset_id":"newcomer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"freshman%1:18:01::","call_ids":["senseval3.d001.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"freshman%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.004705000000000001,"input_uncached_usd":0.0023650000000000003,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4010351250180975}
{"item_id":"senseval3.d001.s023.t005","gold_sense_keys":["republican%1:18:01::"],"candidates":[{"index":1,"sense_key":"republican%1:18:01::","synset_id":"republican.n.01"},{"index":2,"sense_key":"republican%1:18:00::","synset_id":"republican.n.02"},{"index":3,"sense_key":"republican%1:17:00::","synset_id":"republican.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%1:18:01::","call_ids":["senseval3.d001.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004255,"input_uncached_usd":0.0024850000000000002,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.48456795909442}
{"item_id":"senseval3.d001.s023.t006","gold_sense_keys":["bring%2:35:04::"],"candidates":[{"index":1,"sense_key":"bring%2:32:03::","synset_id":"bring.v.11"},{"index":2,"sense_key":"bring%2:36:01::","synset_id":"institute.v.02"},{"index":3,"sense_key":"bring%2:35:04::","synset_id":"bring.v.02"},{"index":4,"sense_key":"bring%2:36:00::","synset_id":"bring.v.03"},{"index":5,"sense_key":"bring%2:35:00::","synset_id":"bring.v.04"},{"index":6,"sense_key":"bring%2:37:05::","synset_id":"bring.v.10"},{"index":7,"sense_key":"bring%2:40:00::","synset_id":"fetch.v.02"},{"index":8,"sense_key":"bring%2:35:02::","synset_id":"bring.v.06"},{"index":9,"sense_key":"bring%2:40:02::","synset_id":"lend.v.01"},{"index":10,"sense_key":"bring%2:38:00::","synset_id":"bring.v.01"},{"index":11,"sense_key":"bring%2:30:00::","synset_id":"bring.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"bring%2:30:00::","call_ids":["senseval3.d001.s023.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"bring%2:30:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":883,"cached_input_tokens":0,"output_tokens":1564,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.051335000000000006,"input_uncached_usd":0.0044150000000000005,"input_cached_usd":0.0,"output_usd":0.04692,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":30.3430554169463}
{"item_id":"senseval3.d001.s023.t007","gold_sense_keys":["office%1:26:00::"],"candidates":[{"index":1,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":2,"sense_key":"office%1:04:02::","synset_id":"position.n.06"},{"index":3,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":4,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":6,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":7,"sense_key":"office%1:04:01::","synset_id":"function.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"office%1:26:00::","call_ids":["senseval3.d001.s023.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"office%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":726,"cached_input_tokens":0,"output_tokens":198,"reasoning_output_tokens":182},"cost":{"currency":"USD","total_usd":0.009570000000000002,"input_uncached_usd":0.0036300000000000004,"input_cached_usd":0.0,"output_usd":0.00594,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.452373790903948}
{"item_id":"senseval3.d001.s023.t008","gold_sense_keys":["revolution%1:11:01::"],"candidates":[{"index":1,"sense_key":"revolution%1:11:00::","synset_id":"rotation.n.03"},{"index":2,"sense_key":"revolution%1:04:00::","synset_id":"revolution.n.02"},{"index":3,"sense_key":"revolution%1:11:01::","synset_id":"revolution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"revolution%1:11:01::","call_ids":["senseval3.d001.s023.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"revolution%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.0050550000000000005,"input_uncached_usd":0.002625,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.003783042076975}
{"item_id":"senseval3.d001.s023.t009","gold_sense_keys":["run%2:33:06::"],"candidates":[{"index":1,"sense_key":"run%2:41:02::","synset_id":"operate.v.01"},{"index":2,"sense_key":"run%2:42:00::","synset_id":"run.v.03"},{"index":3,"sense_key":"run%2:30:03::","synset_id":"run.v.14"},{"index":4,"sense_key":"run%2:38:05::","synset_id":"run.v.29"},{"index":5,"sense_key":"run%2:41:01::","synset_id":"run.v.24"},{"index":6,"sense_key":"run%2:42:02::","synset_id":"run.v.16"},{"index":7,"sense_key":"run%2:38:09::","synset_id":"ply.v.03"},{"index":8,"sense_key":"run%2:35:04::","synset_id":"run.v.13"},{"index":9,"sense_key":"run%2:33:06::","synset_id":"campaign.v.01"},{"index":10,"sense_key":"run%2:42:01::","synset_id":"tend.v.01"},{"index":11,"sense_key":"run%2:38:04::","synset_id":"scat.v.01"},{"index":12,"sense_key":"run%2:42:04::","synset_id":"run.v.05"},{"index":13,"sense_key":"run%2:36:02::","synset_id":"carry.v.15"},{"index":14,"sense_key":"run%2:38:02::","synset_id":"run.v.32"},{"index":15,"sense_key":"run%2:38:00::","synset_id":"run.v.01"},{"index":16,"sense_key":"run%2:30:02::","synset_id":"move.v.13"},{"index":17,"sense_key":"run%2:35:00::","synset_id":"function.v.01"},{"index":18,"sense_key":"run%2:38:11::","synset_id":"run.v.11"},{"index":19,"sense_key":"run%2:35:01::","synset_id":"guide.v.05"},{"index":20,"sense_key":"run%2:40:06::","synset_id":"run.v.25"},{"index":21,"sense_key":"run%2:42:07::","synset_id":"range.v.01"},{"index":22,"sense_key":"run%2:30:00::","synset_id":"melt.v.01"},{"index":23,"sense_key":"run%2:36:00::","synset_id":"run.v.21"},{"index":24,"sense_key":"run%2:38:10::","synset_id":"run.v.26"},{"index":25,"sense_key":"run%2:38:01::","synset_id":"run.v.06"},{"index":26,"sense_key":"run%2:33:01::","synset_id":"race.v.02"},{"index":27,"sense_key":"run%2:30:07::","synset_id":"run.v.15"},{"index":28,"sense_key":"run%2:38:06::","synset_id":"run.v.33"},{"index":29,"sense_key":"run%2:30:04::","synset_id":"run.v.41"},{"index":30,"sense_key":"run%2:30:01::","synset_id":"ladder.v.01"},{"index":31,"sense_key":"run%2:42:08::","synset_id":"run.v.23"},{"index":32,"sense_key":"run%2:42:06::","synset_id":"run.v.18"},{"index":33,"sense_key":"run%2:42:03::","synset_id":"prevail.v.03"},{"index":34,"sense_key":"run%2:42:05::","synset_id":"run.v.30"},{"index":35,"sense_key":"run%2:41:04::","synset_id":"run.v.31"},{"index":36,"sense_key":"run%2:33:02::","synset_id":"hunt.v.01"},{"index":37,"sense_key":"run%2:36:01::","synset_id":"play.v.18"},{"index":38,"sense_key":"run%2:41:00::","synset_id":"run.v.19"},{"index":39,"sense_key":"run%2:38:13::","synset_id":"run.v.34"},{"index":40,"sense_key":"run%2:38:08::","synset_id":"run.v.27"},{"index":41,"sense_key":"run%2:38:03::","synset_id":"run.v.28"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"run%2:33:06::","call_ids":["senseval3.d001.s023.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"run%2:33:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2101,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.013025,"input_uncached_usd":0.010505,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0814605420455337}
{"item_id":"senseval3.d001.s024.t000","gold_sense_keys":["freshman%1:18:01::"],"candidates":[{"index":1,"sense_key":"freshman%1:18:01::","synset_id":"newcomer.n.01"},{"index":2,"sense_key":"freshman%1:18:00::","synset_id":"freshman.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"freshman%1:18:01::","call_ids":["senseval3.d001.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"freshman%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.004710000000000001,"input_uncached_usd":0.0024000000000000002,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0088371669407934}
{"item_id":"senseval3.d001.s024.t001","gold_sense_keys":["place%2:35:00::"],"candidates":[{"index":1,"sense_key":"place%2:32:01::","synset_id":"place.v.05"},{"index":2,"sense_key":"place%2:40:00::","synset_id":"locate.v.03"},{"index":3,"sense_key":"place%2:31:00::","synset_id":"identify.v.01"},{"index":4,"sense_key":"place%2:41:01::","synset_id":"place.v.02"},{"index":5,"sense_key":"place%2:32:00::","synset_id":"place.v.16"},{"index":6,"sense_key":"place%2:31:01::","synset_id":"rate.v.01"},{"index":7,"sense_key":"place%2:33:02::","synset_id":"place.v.15"},{"index":8,"sense_key":"place%2:31:13::","synset_id":"place.v.11"},{"index":9,"sense_key":"place%2:33:00::","synset_id":"station.v.01"},{"index":10,"sense_key":"place%2:41:00::","synset_id":"place.v.09"},{"index":11,"sense_key":"place%2:33:03::","synset_id":"target.v.01"},{"index":12,"sense_key":"place%2:40:03::","synset_id":"invest.v.01"},{"index":13,"sense_key":"place%2:36:00::","synset_id":"set.v.09"},{"index":14,"sense_key":"place%2:31:03::","synset_id":"place.v.06"},{"index":15,"sense_key":"place%2:35:00::","synset_id":"put.v.01"},{"index":16,"sense_key":"place%2:42:00::","synset_id":"place.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"place%2:35:00::","call_ids":["senseval3.d001.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"place%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1034,"cached_input_tokens":0,"output_tokens":235,"reasoning_output_tokens":219},"cost":{"currency":"USD","total_usd":0.012220000000000002,"input_uncached_usd":0.00517,"input_cached_usd":0.0,"output_usd":0.00705,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.336364125018008}
{"item_id":"senseval3.d001.s024.t002","gold_sense_keys":["emphasis%1:07:00::","emphasis%1:26:00::"],"candidates":[{"index":1,"sense_key":"emphasis%1:10:01::","synset_id":"emphasis.n.03"},{"index":2,"sense_key":"emphasis%1:10:00::","synset_id":"stress.n.01"},{"index":3,"sense_key":"emphasis%1:26:00::","synset_id":"emphasis.n.01"},{"index":4,"sense_key":"emphasis%1:07:00::","synset_id":"vehemence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"emphasis%1:26:00::","call_ids":["senseval3.d001.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"emphasis%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":111},"cost":{"currency":"USD","total_usd":0.00677,"input_uncached_usd":0.0029600000000000004,"input_cached_usd":0.0,"output_usd":0.00381,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.796643374953419}
{"item_id":"senseval3.d001.s024.t004","gold_sense_keys":["role%1:04:00::","role%1:04:01::"],"candidates":[{"index":1,"sense_key":"role%1:09:00::","synset_id":"character.n.04"},{"index":2,"sense_key":"role%1:04:01::","synset_id":"role.n.04"},{"index":3,"sense_key":"role%1:07:00::","synset_id":"function.n.02"},{"index":4,"sense_key":"role%1:04:00::","synset_id":"function.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"role%1:04:00::","call_ids":["senseval3.d001.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"role%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018845,"input_uncached_usd":0.0030050000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.453835249994881}
{"item_id":"senseval3.d001.s024.t005","gold_sense_keys":["spread%2:32:00::","spread%2:35:00::"],"candidates":[{"index":1,"sense_key":"spread%2:42:00::","synset_id":"spread.v.03"},{"index":2,"sense_key":"spread%2:35:13::","synset_id":"spread.v.10"},{"index":3,"sense_key":"spread%2:35:00::","synset_id":"spread.v.01"},{"index":4,"sense_key":"spread%2:38:02::","synset_id":"diffuse.v.01"},{"index":5,"sense_key":"spread%2:32:02::","synset_id":"go_around.v.02"},{"index":6,"sense_key":"spread%2:32:00::","synset_id":"circulate.v.02"},{"index":7,"sense_key":"spread%2:35:07::","synset_id":"spread.v.02"},{"index":8,"sense_key":"spread%2:35:03::","synset_id":"unfold.v.04"},{"index":9,"sense_key":"spread%2:35:12::","synset_id":"spread.v.09"},{"index":10,"sense_key":"spread%2:38:00::","synset_id":"spread.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"spread%2:35:00::","call_ids":["senseval3.d001.s024.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"spread%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":816,"cached_input_tokens":0,"output_tokens":471,"reasoning_output_tokens":455},"cost":{"currency":"USD","total_usd":0.01821,"input_uncached_usd":0.00408,"input_cached_usd":0.0,"output_usd":0.01413,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.274882249999791}
{"item_id":"senseval3.d001.s024.t006","gold_sense_keys":["revolution%1:11:01::"],"candidates":[{"index":1,"sense_key":"revolution%1:11:00::","synset_id":"rotation.n.03"},{"index":2,"sense_key":"revolution%1:11:01::","synset_id":"revolution.n.01"},{"index":3,"sense_key":"revolution%1:04:00::","synset_id":"revolution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"revolution%1:11:01::","call_ids":["senseval3.d001.s024.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"revolution%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":80},"cost":{"currency":"USD","total_usd":0.00551,"input_uncached_usd":0.00266,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3174023339524865}
{"item_id":"senseval3.d001.s024.t007","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":2,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":5,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":6,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":676,"cached_input_tokens":0,"output_tokens":166,"reasoning_output_tokens":151},"cost":{"currency":"USD","total_usd":0.00836,"input_uncached_usd":0.00338,"input_cached_usd":0.0,"output_usd":0.00498,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.836622416973114}
{"item_id":"senseval3.d001.s024.t008","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval3.d001.s024.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":539,"cached_input_tokens":0,"output_tokens":358,"reasoning_output_tokens":343},"cost":{"currency":"USD","total_usd":0.013435000000000002,"input_uncached_usd":0.0026950000000000003,"input_cached_usd":0.0,"output_usd":0.01074,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.803128624917008}
{"item_id":"senseval3.d001.s024.t009","gold_sense_keys":["making%1:04:00::"],"candidates":[{"index":1,"sense_key":"making%1:06:00::","synset_id":"making.n.03"},{"index":2,"sense_key":"making%1:07:00::","synset_id":"qualification.n.01"},{"index":3,"sense_key":"making%1:04:00::","synset_id":"devising.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"making%1:04:00::","call_ids":["senseval3.d001.s024.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"making%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.005135000000000001,"input_uncached_usd":0.002795,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.682562040979974}
{"item_id":"senseval3.d001.s025.t002","gold_sense_keys":["indicate%2:32:02::"],"candidates":[{"index":1,"sense_key":"indicate%2:32:01::","synset_id":"bespeak.v.01"},{"index":2,"sense_key":"indicate%2:32:00::","synset_id":"indicate.v.03"},{"index":3,"sense_key":"indicate%2:32:04::","synset_id":"indicate.v.05"},{"index":4,"sense_key":"indicate%2:32:03::","synset_id":"indicate.v.02"},{"index":5,"sense_key":"indicate%2:32:02::","synset_id":"argue.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"indicate%2:32:03::","call_ids":["senseval3.d001.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"indicate%2:32:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01889,"input_uncached_usd":0.00305,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.515584417036735}
{"item_id":"senseval3.d001.s025.t006","gold_sense_keys":["attract%2:35:00::"],"candidates":[{"index":1,"sense_key":"attract%2:35:00::","synset_id":"attract.v.01"},{"index":2,"sense_key":"attract%2:35:01::","synset_id":"attract.v.03"},{"index":3,"sense_key":"attract%2:37:00::","synset_id":"attract.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attract%2:35:00::","call_ids":["senseval3.d001.s025.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attract%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0185,"input_uncached_usd":0.00266,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.53068862494547}
{"item_id":"senseval3.d001.s025.t008","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"},{"index":2,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s025.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01805,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":17.499275833019055}
{"item_id":"senseval3.d001.s025.t010","gold_sense_keys":["office%1:26:00::"],"candidates":[{"index":1,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":2,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":3,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":4,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":5,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":6,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":7,"sense_key":"office%1:04:02::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"office%1:04:02::","call_ids":["senseval3.d001.s025.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"office%1:04:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":119},"cost":{"currency":"USD","total_usd":0.0075250000000000004,"input_uncached_usd":0.0034750000000000002,"input_cached_usd":0.0,"output_usd":0.00405,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5346197079634294}
{"item_id":"senseval3.d001.s026.t000","gold_sense_keys":["potential%5:00:00:prospective:00"],"candidates":[{"index":1,"sense_key":"potential%5:00:00:prospective:00","synset_id":"likely.s.01"},{"index":2,"sense_key":"potential%3:00:00::","synset_id":"potential.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"potential%5:00:00:prospective:00","call_ids":["senseval3.d001.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"potential%5:00:00:prospective:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":436,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":116},"cost":{"currency":"USD","total_usd":0.00614,"input_uncached_usd":0.00218,"input_cached_usd":0.0,"output_usd":0.00396,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9986810829723254}
{"item_id":"senseval3.d001.s026.t001","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"candidate%1:18:00::","call_ids":["senseval3.d001.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"candidate%1:18:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":141},"cost":{"currency":"USD","total_usd":0.006875000000000001,"input_uncached_usd":0.0021650000000000003,"input_cached_usd":0.0,"output_usd":0.00471,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1045545000815764}
{"item_id":"senseval3.d001.s026.t002","gold_sense_keys":["discourage%2:37:00::"],"candidates":[{"index":1,"sense_key":"discourage%2:32:02::","synset_id":"warn.v.02"},{"index":2,"sense_key":"discourage%2:37:00::","synset_id":"discourage.v.02"},{"index":3,"sense_key":"discourage%2:32:01::","synset_id":"deter.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"discourage%2:32:01::","call_ids":["senseval3.d001.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"discourage%2:32:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":327,"reasoning_output_tokens":311},"cost":{"currency":"USD","total_usd":0.012289999999999999,"input_uncached_usd":0.00248,"input_cached_usd":0.0,"output_usd":0.009810000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.483675791998394}
{"item_id":"senseval3.d001.s026.t003","gold_sense_keys":["run%2:33:06::"],"candidates":[{"index":1,"sense_key":"run%2:38:11::","synset_id":"run.v.11"},{"index":2,"sense_key":"run%2:41:04::","synset_id":"run.v.31"},{"index":3,"sense_key":"run%2:40:06::","synset_id":"run.v.25"},{"index":4,"sense_key":"run%2:38:00::","synset_id":"run.v.01"},{"index":5,"sense_key":"run%2:30:00::","synset_id":"melt.v.01"},{"index":6,"sense_key":"run%2:30:01::","synset_id":"ladder.v.01"},{"index":7,"sense_key":"run%2:35:04::","synset_id":"run.v.13"},{"index":8,"sense_key":"run%2:42:04::","synset_id":"run.v.05"},{"index":9,"sense_key":"run%2:36:02::","synset_id":"carry.v.15"},{"index":10,"sense_key":"run%2:38:01::","synset_id":"run.v.06"},{"index":11,"sense_key":"run%2:41:02::","synset_id":"operate.v.01"},{"index":12,"sense_key":"run%2:33:06::","synset_id":"campaign.v.01"},{"index":13,"sense_key":"run%2:42:03::","synset_id":"prevail.v.03"},{"index":14,"sense_key":"run%2:38:05::","synset_id":"run.v.29"},{"index":15,"sense_key":"run%2:38:08::","synset_id":"run.v.27"},{"index":16,"sense_key":"run%2:42:01::","synset_id":"tend.v.01"},{"index":17,"sense_key":"run%2:30:04::","synset_id":"run.v.41"},{"index":18,"sense_key":"run%2:38:03::","synset_id":"run.v.28"},{"index":19,"sense_key":"run%2:42:08::","synset_id":"run.v.23"},{"index":20,"sense_key":"run%2:42:07::","synset_id":"range.v.01"},{"index":21,"sense_key":"run%2:41:01::","synset_id":"run.v.24"},{"index":22,"sense_key":"run%2:42:00::","synset_id":"run.v.03"},{"index":23,"sense_key":"run%2:33:02::","synset_id":"hunt.v.01"},{"index":24,"sense_key":"run%2:41:00::","synset_id":"run.v.19"},{"index":25,"sense_key":"run%2:38:06::","synset_id":"run.v.33"},{"index":26,"sense_key":"run%2:38:02::","synset_id":"run.v.32"},{"index":27,"sense_key":"run%2:36:00::","synset_id":"run.v.21"},{"index":28,"sense_key":"run%2:33:01::","synset_id":"race.v.02"},{"index":29,"sense_key":"run%2:35:01::","synset_id":"guide.v.05"},{"index":30,"sense_key":"run%2:35:00::","synset_id":"function.v.01"},{"index":31,"sense_key":"run%2:38:09::","synset_id":"ply.v.03"},{"index":32,"sense_key":"run%2:38:04::","synset_id":"scat.v.01"},{"index":33,"sense_key":"run%2:42:06::","synset_id":"run.v.18"},{"index":34,"sense_key":"run%2:30:07::","synset_id":"run.v.15"},{"index":35,"sense_key":"run%2:38:13::","synset_id":"run.v.34"},{"index":36,"sense_key":"run%2:38:10::","synset_id":"run.v.26"},{"index":37,"sense_key":"run%2:36:01::","synset_id":"play.v.18"},{"index":38,"sense_key":"run%2:42:02::","synset_id":"run.v.16"},{"index":39,"sense_key":"run%2:30:03::","synset_id":"run.v.14"},{"index":40,"sense_key":"run%2:30:02::","synset_id":"move.v.13"},{"index":41,"sense_key":"run%2:42:05::","synset_id":"run.v.30"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"run%2:33:06::","call_ids":["senseval3.d001.s026.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"run%2:33:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2062,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.013400000000000002,"input_uncached_usd":0.010310000000000001,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6111249580280855}
{"item_id":"senseval3.d001.s026.t006","gold_sense_keys":["prospect%1:09:01::"],"candidates":[{"index":1,"sense_key":"prospect%1:04:00::","synset_id":"prognosis.n.02"},{"index":2,"sense_key":"prospect%1:09:01::","synset_id":"expectation.n.01"},{"index":3,"sense_key":"prospect%1:26:00::","synset_id":"prospect.n.01"},{"index":4,"sense_key":"prospect%1:09:00::","synset_id":"view.n.02"},{"index":5,"sense_key":"prospect%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"prospect%1:09:01::","call_ids":["senseval3.d001.s026.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"prospect%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":168,"reasoning_output_tokens":152},"cost":{"currency":"USD","total_usd":0.007825,"input_uncached_usd":0.002785,"input_cached_usd":0.0,"output_usd":0.00504,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.107693874975666}
{"item_id":"senseval3.d001.s026.t007","gold_sense_keys":["defeat%1:11:00::"],"candidates":[{"index":1,"sense_key":"defeat%1:11:00::","synset_id":"defeat.n.01"},{"index":2,"sense_key":"defeat%1:12:00::","synset_id":"frustration.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"defeat%1:11:00::","call_ids":["senseval3.d001.s026.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"defeat%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.003735,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.50910670903977}
{"item_id":"senseval3.d001.s026.t009","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s026.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.004125,"input_uncached_usd":0.0025050000000000003,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.055469083948992}
{"item_id":"senseval3.d001.s026.t010","gold_sense_keys":["opponent%1:18:02::"],"candidates":[{"index":1,"sense_key":"opponent%1:18:02::","synset_id":"opposition.n.04"},{"index":2,"sense_key":"opponent%1:18:00::","synset_id":"adversary.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"opponent%1:18:02::","call_ids":["senseval3.d001.s026.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"opponent%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":430,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.00536,"input_uncached_usd":0.00215,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.122120584012009}
{"item_id":"senseval3.d001.s027.t000","gold_sense_keys":["extent%1:26:00::"],"candidates":[{"index":1,"sense_key":"extent%1:07:00::","synset_id":"extent.n.02"},{"index":2,"sense_key":"extent%1:26:00::","synset_id":"extent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"extent%1:26:00::","call_ids":["senseval3.d001.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"extent%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.00385,"input_uncached_usd":0.0022,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.433709540986456}
{"item_id":"senseval3.d001.s027.t001","gold_sense_keys":["potential%5:00:00:prospective:00"],"candidates":[{"index":1,"sense_key":"potential%3:00:00::","synset_id":"potential.a.01"},{"index":2,"sense_key":"potential%5:00:00:prospective:00","synset_id":"likely.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"potential%5:00:00:prospective:00","call_ids":["senseval3.d001.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"potential%5:00:00:prospective:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":431,"cached_input_tokens":0,"output_tokens":229,"reasoning_output_tokens":213},"cost":{"currency":"USD","total_usd":0.009025,"input_uncached_usd":0.0021550000000000002,"input_cached_usd":0.0,"output_usd":0.00687,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.3672034160699695}
{"item_id":"senseval3.d001.s027.t002","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"candidate%1:18:00::","call_ids":["senseval3.d001.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"candidate%1:18:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":134},"cost":{"currency":"USD","total_usd":0.00664,"input_uncached_usd":0.00214,"input_cached_usd":0.0,"output_usd":0.0045000000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3258082079701126}
{"item_id":"senseval3.d001.s027.t004","gold_sense_keys":["realize%2:31:00::","realize%2:31:01::"],"candidates":[{"index":1,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"},{"index":2,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"},{"index":3,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"},{"index":4,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"},{"index":5,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":6,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"realize%2:31:01::","call_ids":["senseval3.d001.s027.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"realize%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":256,"reasoning_output_tokens":240},"cost":{"currency":"USD","total_usd":0.010950000000000001,"input_uncached_usd":0.0032700000000000003,"input_cached_usd":0.0,"output_usd":0.00768,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.627052083960734}
{"item_id":"senseval3.d001.s027.t007","gold_sense_keys":["work%2:41:03::"],"candidates":[{"index":1,"sense_key":"work%2:36:01::","synset_id":"bring.v.03"},{"index":2,"sense_key":"work%2:30:10::","synset_id":"work.v.27"},{"index":3,"sense_key":"work%2:36:12::","synset_id":"shape.v.02"},{"index":4,"sense_key":"work%2:35:00::","synset_id":"knead.v.01"},{"index":5,"sense_key":"work%2:34:10::","synset_id":"exploit.v.01"},{"index":6,"sense_key":"work%2:30:14::","synset_id":"ferment.v.03"},{"index":7,"sense_key":"work%2:41:12::","synset_id":"work.v.17"},{"index":8,"sense_key":"work%2:30:13::","synset_id":"sour.v.01"},{"index":9,"sense_key":"work%2:41:10::","synset_id":"influence.v.01"},{"index":10,"sense_key":"work%2:41:05::","synset_id":"work.v.09"},{"index":11,"sense_key":"work%2:39:10::","synset_id":"work.v.18"},{"index":12,"sense_key":"work%2:38:02::","synset_id":"work.v.10"},{"index":13,"sense_key":"work%2:41:01::","synset_id":"work.v.12"},{"index":14,"sense_key":"work%2:37:10::","synset_id":"work.v.19"},{"index":15,"sense_key":"work%2:41:03::","synset_id":"work.v.03"},{"index":16,"sense_key":"work%2:35:14::","synset_id":"work.v.21"},{"index":17,"sense_key":"work%2:36:00::","synset_id":"work.v.05"},{"index":18,"sense_key":"work%2:41:00::","synset_id":"work.v.02"},{"index":19,"sense_key":"work%2:36:09::","synset_id":"cultivate.v.02"},{"index":20,"sense_key":"work%2:41:11::","synset_id":"work.v.16"},{"index":21,"sense_key":"work%2:35:02::","synset_id":"function.v.01"},{"index":22,"sense_key":"work%2:29:00::","synset_id":"exercise.v.03"},{"index":23,"sense_key":"work%2:31:13::","synset_id":"solve.v.01"},{"index":24,"sense_key":"work%2:41:02::","synset_id":"work.v.01"},{"index":25,"sense_key":"work%2:38:00::","synset_id":"make.v.36"},{"index":26,"sense_key":"work%2:42:10::","synset_id":"work.v.14"},{"index":27,"sense_key":"work%2:41:04::","synset_id":"work.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"work%2:41:03::","call_ids":["senseval3.d001.s027.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"work%2:41:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1533,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":134},"cost":{"currency":"USD","total_usd":0.012165000000000002,"input_uncached_usd":0.007665000000000001,"input_cached_usd":0.0,"output_usd":0.0045000000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.01786608400289}
{"item_id":"senseval3.d001.s027.t009","gold_sense_keys":["hinder%2:41:00::"],"candidates":[{"index":1,"sense_key":"hinder%2:41:00::","synset_id":"obstruct.v.01"},{"index":2,"sense_key":"hinder%2:33:00::","synset_id":"handicap.v.03"},{"index":3,"sense_key":"hinder%2:41:01::","synset_id":"impede.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"hinder%2:33:00::","call_ids":["senseval3.d001.s027.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"hinder%2:33:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":453,"reasoning_output_tokens":437},"cost":{"currency":"USD","total_usd":0.01607,"input_uncached_usd":0.00248,"input_cached_usd":0.0,"output_usd":0.01359,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.906067625037394}
{"item_id":"senseval3.d001.s027.t010","gold_sense_keys":["attempt%1:04:00::"],"candidates":[{"index":1,"sense_key":"attempt%1:04:02::","synset_id":"attack.n.05"},{"index":2,"sense_key":"attempt%1:04:00::","synset_id":"attempt.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"attempt%1:04:00::","call_ids":["senseval3.d001.s027.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"attempt%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0040100000000000005,"input_uncached_usd":0.0023000000000000004,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4995868749683723}
{"item_id":"senseval3.d001.s027.t012","gold_sense_keys":["slate%1:10:02::"],"candidates":[{"index":1,"sense_key":"slate%1:27:01::","synset_id":"slate.n.02"},{"index":2,"sense_key":"slate%1:06:00::","synset_id":"slate.n.01"},{"index":3,"sense_key":"slate%1:27:00::","synset_id":"slate.n.03"},{"index":4,"sense_key":"slate%1:10:02::","synset_id":"slate.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"slate%1:10:02::","call_ids":["senseval3.d001.s027.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"slate%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.004125,"input_uncached_usd":0.002415,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3879519579932094}
{"item_id":"senseval3.d001.s027.t014","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s027.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":114},"cost":{"currency":"USD","total_usd":0.006005,"input_uncached_usd":0.002135,"input_cached_usd":0.0,"output_usd":0.00387,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2129303329857066}
{"item_id":"senseval3.d001.s028.t003","gold_sense_keys":["partial%5:00:00:incomplete:00"],"candidates":[{"index":1,"sense_key":"partial%5:00:00:incomplete:00","synset_id":"partial.s.01"},{"index":2,"sense_key":"partial%5:00:00:inclined:02","synset_id":"fond.s.03"},{"index":3,"sense_key":"partial%3:00:01::","synset_id":"partial.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"partial%5:00:00:incomplete:00","call_ids":["senseval3.d001.s028.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"partial%5:00:00:incomplete:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.0043,"input_uncached_usd":0.00253,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.491579958004877}
{"item_id":"senseval3.d001.s028.t004","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::"],"candidates":[{"index":1,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":2,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":3,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"},{"index":4,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":5,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":6,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"reason%1:10:01::","call_ids":["senseval3.d001.s028.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"reason%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":175,"reasoning_output_tokens":159},"cost":{"currency":"USD","total_usd":0.00861,"input_uncached_usd":0.00336,"input_cached_usd":0.0,"output_usd":0.00525,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.000530334073119}
{"item_id":"senseval3.d001.s029.t000","gold_sense_keys":["extent%1:26:00::"],"candidates":[{"index":1,"sense_key":"extent%1:07:00::","synset_id":"extent.n.02"},{"index":2,"sense_key":"extent%1:26:00::","synset_id":"extent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"extent%1:26:00::","call_ids":["senseval3.d001.s029.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"extent%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.0045249999999999995,"input_uncached_usd":0.002275,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.03442554094363}
{"item_id":"senseval3.d001.s029.t001","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":2,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s029.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.004745000000000001,"input_uncached_usd":0.0025550000000000004,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6855178750120103}
{"item_id":"senseval3.d001.s029.t003","gold_sense_keys":["hold%2:40:00::"],"candidates":[{"index":1,"sense_key":"hold%2:31:01::","synset_id":"deem.v.01"},{"index":2,"sense_key":"hold%2:34:00::","synset_id":"carry.v.33"},{"index":3,"sense_key":"hold%2:32:15::","synset_id":"agree.v.01"},{"index":4,"sense_key":"hold%2:35:00::","synset_id":"hold.v.02"},{"index":5,"sense_key":"hold%2:41:14::","synset_id":"hold.v.29"},{"index":6,"sense_key":"hold%2:32:13::","synset_id":"oblige.v.02"},{"index":7,"sense_key":"hold%2:42:12::","synset_id":"hold.v.26"},{"index":8,"sense_key":"hold%2:40:04::","synset_id":"bear.v.11"},{"index":9,"sense_key":"hold%2:42:01::","synset_id":"hold.v.13"},{"index":10,"sense_key":"hold%2:42:04::","synset_id":"apply.v.02"},{"index":11,"sense_key":"hold%2:42:00::","synset_id":"keep.v.01"},{"index":12,"sense_key":"hold%2:42:05::","synset_id":"accommodate.v.04"},{"index":13,"sense_key":"hold%2:33:00::","synset_id":"hold.v.33"},{"index":14,"sense_key":"hold%2:31:10::","synset_id":"hold.v.23"},{"index":15,"sense_key":"hold%2:31:04::","synset_id":"hold.v.22"},{"index":16,"sense_key":"hold%2:40:02::","synset_id":"retain.v.03"},{"index":17,"sense_key":"hold%2:42:07::","synset_id":"hold.v.17"},{"index":18,"sense_key":"hold%2:41:00::","synset_id":"reserve.v.04"},{"index":19,"sense_key":"hold%2:41:13::","synset_id":"hold.v.28"},{"index":20,"sense_key":"hold%2:35:03::","synset_id":"restrain.v.03"},{"index":21,"sense_key":"hold%2:40:00::","synset_id":"have.v.01"},{"index":22,"sense_key":"hold%2:38:05::","synset_id":"halt.v.01"},{"index":23,"sense_key":"hold%2:35:01::","synset_id":"hold.v.10"},{"index":24,"sense_key":"hold%2:29:04::","synset_id":"hold.v.36"},{"index":25,"sense_key":"hold%2:35:02::","synset_id":"hold.v.14"},{"index":26,"sense_key":"hold%2:41:15::","synset_id":"control.v.02"},{"index":27,"sense_key":"hold%2:42:08::","synset_id":"defy.v.01"},{"index":28,"sense_key":"hold%2:42:14::","synset_id":"contain.v.05"},{"index":29,"sense_key":"hold%2:33:05::","synset_id":"defend.v.03"},{"index":30,"sense_key":"hold%2:36:00::","synset_id":"hold.v.03"},{"index":31,"sense_key":"hold%2:37:00::","synset_id":"harbor.v.01"},{"index":32,"sense_key":"hold%2:31:08::","synset_id":"hold.v.16"},{"index":33,"sense_key":"hold%2:42:13::","synset_id":"hold.v.11"},{"index":34,"sense_key":"hold%2:32:11::","synset_id":"declare.v.04"},{"index":35,"sense_key":"hold%2:35:09::","synset_id":"hold.v.31"},{"index":36,"sense_key":"hold%2:42:02::","synset_id":"prevail.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":21,"chosen_sense_key":"hold%2:40:00::","call_ids":["senseval3.d001.s029.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":21,"predicted_sense_key":"hold%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2000,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":120},"cost":{"currency":"USD","total_usd":0.01408,"input_uncached_usd":0.01,"input_cached_usd":0.0,"output_usd":0.00408,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2617056249873713}
{"item_id":"senseval3.d001.s029.t005","gold_sense_keys":["share%1:04:00::"],"candidates":[{"index":1,"sense_key":"share%1:21:01::","synset_id":"share.n.02"},{"index":2,"sense_key":"share%1:06:00::","synset_id":"plowshare.n.01"},{"index":3,"sense_key":"share%1:04:00::","synset_id":"parcel.n.02"},{"index":4,"sense_key":"share%1:21:00::","synset_id":"share.n.01"},{"index":5,"sense_key":"share%1:04:01::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"share%1:04:00::","call_ids":["senseval3.d001.s029.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"share%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":389,"reasoning_output_tokens":373},"cost":{"currency":"USD","total_usd":0.01476,"input_uncached_usd":0.0030900000000000003,"input_cached_usd":0.0,"output_usd":0.01167,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.890202041948214}
{"item_id":"senseval3.d001.s029.t006","gold_sense_keys":["power%1:07:00::"],"candidates":[{"index":1,"sense_key":"power%1:26:00::","synset_id":"office.n.04"},{"index":2,"sense_key":"power%1:14:00::","synset_id":"world_power.n.01"},{"index":3,"sense_key":"power%1:19:00::","synset_id":"power.n.02"},{"index":4,"sense_key":"power%1:18:00::","synset_id":"power.n.05"},{"index":5,"sense_key":"power%1:07:00::","synset_id":"power.n.01"},{"index":6,"sense_key":"power%1:07:02::","synset_id":"might.n.01"},{"index":7,"sense_key":"power%1:09:01::","synset_id":"ability.n.02"},{"index":8,"sense_key":"power%1:10:00::","synset_id":"exponent.n.03"},{"index":9,"sense_key":"power%1:18:01::","synset_id":"baron.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"power%1:07:00::","call_ids":["senseval3.d001.s029.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"power%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":752,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.007300000000000001,"input_uncached_usd":0.0037600000000000003,"input_cached_usd":0.0,"output_usd":0.00354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.513657624949701}
{"item_id":"senseval3.d001.s029.t007","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"},{"index":2,"sense_key":"able%3:00:00::","synset_id":"able.a.01"},{"index":3,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"},{"index":4,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"able%3:00:00::","call_ids":["senseval3.d001.s029.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"able%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.005050000000000001,"input_uncached_usd":0.0028000000000000004,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8255824579391629}
{"item_id":"senseval3.d001.s029.t008","gold_sense_keys":["translate%2:30:01::"],"candidates":[{"index":1,"sense_key":"translate%2:35:00::","synset_id":"translate.v.04"},{"index":2,"sense_key":"translate%2:30:00::","synset_id":"translate.v.05"},{"index":3,"sense_key":"translate%2:32:05::","synset_id":"translate.v.10"},{"index":4,"sense_key":"translate%2:32:00::","synset_id":"translate.v.01"},{"index":5,"sense_key":"translate%2:42:00::","synset_id":"translate.v.07"},{"index":6,"sense_key":"translate%2:30:01::","synset_id":"translate.v.02"},{"index":7,"sense_key":"translate%2:31:00::","synset_id":"understand.v.03"},{"index":8,"sense_key":"translate%2:38:04::","synset_id":"translate.v.08"},{"index":9,"sense_key":"translate%2:32:06::","synset_id":"translate.v.09"},{"index":10,"sense_key":"translate%2:42:03::","synset_id":"translate.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"translate%2:30:01::","call_ids":["senseval3.d001.s029.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"translate%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":781,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":119},"cost":{"currency":"USD","total_usd":0.007955,"input_uncached_usd":0.0039050000000000005,"input_cached_usd":0.0,"output_usd":0.00405,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7399409579811618}
{"item_id":"senseval3.d001.s029.t010","gold_sense_keys":["local%3:01:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:02::","synset_id":"local.a.03"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:01::","synset_id":"local.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s029.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0183,"input_uncached_usd":0.0024600000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.358418332994916}
{"item_id":"senseval3.d001.s029.t011","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"},{"index":3,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"benefit%1:07:00::","call_ids":["senseval3.d001.s029.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.004525,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9296370840165764}
{"item_id":"senseval3.d001.s029.t015","gold_sense_keys":["have%2:40:00::","have%2:42:12::"],"candidates":[{"index":1,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":2,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":3,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":4,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":6,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":7,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":8,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":9,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":10,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":11,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":12,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":13,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":14,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":15,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":16,"sense_key":"have%2:35:00::","synset_id":"take.v.35"},{"index":17,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":18,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":19,"sense_key":"have%2:29:02::","synset_id":"have.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d001.s029.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1209,"cached_input_tokens":0,"output_tokens":180,"reasoning_output_tokens":165},"cost":{"currency":"USD","total_usd":0.011445,"input_uncached_usd":0.006045000000000001,"input_cached_usd":0.0,"output_usd":0.0054,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6215921249240637}
{"item_id":"senseval3.d001.s029.t016","gold_sense_keys":["strong%3:00:00::"],"candidates":[{"index":1,"sense_key":"strong%5:00:00:powerful:00","synset_id":"potent.s.02"},{"index":2,"sense_key":"strong%5:00:00:alcoholic:00","synset_id":"hard.s.04"},{"index":3,"sense_key":"strong%5:00:00:sound:01","synset_id":"solid.s.05"},{"index":4,"sense_key":"strong%5:00:00:fresh:01","synset_id":"strong.s.07"},{"index":5,"sense_key":"strong%5:00:00:forceful:00","synset_id":"firm.s.03"},{"index":6,"sense_key":"strong%3:00:04::","synset_id":"potent.a.03"},{"index":7,"sense_key":"strong%5:00:00:invulnerable:00","synset_id":"impregnable.s.01"},{"index":8,"sense_key":"strong%5:00:00:intense:00","synset_id":"strong.s.01"},{"index":9,"sense_key":"strong%5:00:00:irregular:00","synset_id":"strong.s.05"},{"index":10,"sense_key":"strong%3:00:00::","synset_id":"strong.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"strong%3:00:00::","call_ids":["senseval3.d001.s029.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"strong%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":806,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01987,"input_uncached_usd":0.004030000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.279269916936755}
{"item_id":"senseval3.d001.s029.t017","gold_sense_keys":["incentive%1:16:00::"],"candidates":[{"index":1,"sense_key":"incentive%1:21:00::","synset_id":"bonus.n.02"},{"index":2,"sense_key":"incentive%1:16:00::","synset_id":"incentive.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"incentive%1:16:00::","call_ids":["senseval3.d001.s029.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"incentive%1:16:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.003985000000000001,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6926302920328453}
{"item_id":"senseval3.d001.s029.t018","gold_sense_keys":["keep%2:41:02::"],"candidates":[{"index":1,"sense_key":"keep%2:30:10::","synset_id":"preserve.v.04"},{"index":2,"sense_key":"keep%2:42:02::","synset_id":"keep.v.15"},{"index":3,"sense_key":"keep%2:41:02::","synset_id":"retain.v.02"},{"index":4,"sense_key":"keep%2:42:07::","synset_id":"continue.v.01"},{"index":5,"sense_key":"keep%2:41:03::","synset_id":"observe.v.09"},{"index":6,"sense_key":"keep%2:42:03::","synset_id":"keep.v.12"},{"index":7,"sense_key":"keep%2:40:00::","synset_id":"keep.v.03"},{"index":8,"sense_key":"keep%2:42:00::","synset_id":"keep.v.01"},{"index":9,"sense_key":"keep%2:40:10::","synset_id":"keep.v.20"},{"index":10,"sense_key":"keep%2:40:13::","synset_id":"keep.v.18"},{"index":11,"sense_key":"keep%2:41:00::","synset_id":"restrain.v.01"},{"index":12,"sense_key":"keep%2:40:12::","synset_id":"keep.v.19"},{"index":13,"sense_key":"keep%2:41:01::","synset_id":"prevent.v.02"},{"index":14,"sense_key":"keep%2:42:01::","synset_id":"keep.v.09"},{"index":15,"sense_key":"keep%2:42:12::","synset_id":"keep.v.16"},{"index":16,"sense_key":"keep%2:40:09::","synset_id":"keep_open.v.01"},{"index":17,"sense_key":"keep%2:31:00::","synset_id":"observe.v.08"},{"index":18,"sense_key":"keep%2:34:00::","synset_id":"sustain.v.04"},{"index":19,"sense_key":"keep%2:35:10::","synset_id":"keep.v.21"},{"index":20,"sense_key":"keep%2:41:05::","synset_id":"observe.v.06"},{"index":21,"sense_key":"keep%2:32:00::","synset_id":"keep.v.08"},{"index":22,"sense_key":"keep%2:40:01::","synset_id":"keep.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"keep%2:41:02::","call_ids":["senseval3.d001.s029.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"keep%2:41:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1304,"cached_input_tokens":0,"output_tokens":166,"reasoning_output_tokens":150},"cost":{"currency":"USD","total_usd":0.011500000000000002,"input_uncached_usd":0.006520000000000001,"input_cached_usd":0.0,"output_usd":0.00498,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7862994159804657}
{"item_id":"senseval3.d001.s029.t020","gold_sense_keys":["democrat%1:18:01::"],"candidates":[{"index":1,"sense_key":"democrat%1:18:00::","synset_id":"democrat.n.02"},{"index":2,"sense_key":"democrat%1:18:01::","synset_id":"democrat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democrat%1:18:01::","call_ids":["senseval3.d001.s029.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democrat%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004110000000000001,"input_uncached_usd":0.0021300000000000004,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0419211250264198}
{"item_id":"senseval3.d001.s029.t021","gold_sense_keys":["office%1:26:00::"],"candidates":[{"index":1,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":2,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":3,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":4,"sense_key":"office%1:04:02::","synset_id":"position.n.06"},{"index":5,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":6,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":7,"sense_key":"office%1:14:00::","synset_id":"office.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"office%1:26:00::","call_ids":["senseval3.d001.s029.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"office%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":696,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.0054600000000000004,"input_uncached_usd":0.0034800000000000005,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5575645419303328}
{"item_id":"senseval3.d001.s030.t001","gold_sense_keys":["suggest%2:32:02::"],"candidates":[{"index":1,"sense_key":"suggest%2:32:01::","synset_id":"suggest.v.05"},{"index":2,"sense_key":"suggest%2:32:04::","synset_id":"hint.v.01"},{"index":3,"sense_key":"suggest%2:32:02::","synset_id":"suggest.v.03"},{"index":4,"sense_key":"suggest%2:32:03::","synset_id":"indicate.v.05"},{"index":5,"sense_key":"suggest%2:32:00::","synset_id":"propose.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"suggest%2:32:02::","call_ids":["senseval3.d001.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"suggest%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.0057150000000000005,"input_uncached_usd":0.0028950000000000004,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4340943329734728}
{"item_id":"senseval3.d001.s030.t002","gold_sense_keys":["republican%1:18:01::"],"candidates":[{"index":1,"sense_key":"republican%1:18:00::","synset_id":"republican.n.02"},{"index":2,"sense_key":"republican%1:18:01::","synset_id":"republican.n.01"},{"index":3,"sense_key":"republican%1:17:00::","synset_id":"republican.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"republican%1:18:01::","call_ids":["senseval3.d001.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"republican%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.00419,"input_uncached_usd":0.0024200000000000003,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7323300420539454}
{"item_id":"senseval3.d001.s030.t003","gold_sense_keys":["fare%2:42:00::"],"candidates":[{"index":1,"sense_key":"fare%2:34:00::","synset_id":"fare.v.02"},{"index":2,"sense_key":"fare%2:42:00::","synset_id":"do.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fare%2:42:00::","call_ids":["senseval3.d001.s030.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fare%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.0043,"input_uncached_usd":0.00232,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.207313791033812}
{"item_id":"senseval3.d001.s030.t004","gold_sense_keys":["race%1:11:01::"],"candidates":[{"index":1,"sense_key":"race%1:11:00::","synset_id":"race.n.02"},{"index":2,"sense_key":"race%1:11:01::","synset_id":"race.n.01"},{"index":3,"sense_key":"race%1:14:01::","synset_id":"subspecies.n.01"},{"index":4,"sense_key":"race%1:14:00::","synset_id":"race.n.03"},{"index":5,"sense_key":"race%1:19:00::","synset_id":"slipstream.n.01"},{"index":6,"sense_key":"race%1:06:00::","synset_id":"raceway.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"race%1:11:01::","call_ids":["senseval3.d001.s030.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"race%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.004765,"input_uncached_usd":0.0030550000000000004,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3668419999303296}
{"item_id":"senseval3.d001.s030.t005","gold_sense_keys":["campaign%1:11:00::"],"candidates":[{"index":1,"sense_key":"campaign%1:04:00::","synset_id":"campaign.n.03"},{"index":2,"sense_key":"campaign%1:04:01::","synset_id":"campaign.n.04"},{"index":3,"sense_key":"campaign%1:04:02::","synset_id":"campaign.n.02"},{"index":4,"sense_key":"campaign%1:11:00::","synset_id":"political_campaign.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"campaign%1:11:00::","call_ids":["senseval3.d001.s030.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"campaign%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.0054800000000000005,"input_uncached_usd":0.00287,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2754035419784486}
{"item_id":"senseval3.d001.s031.t000","gold_sense_keys":["local%3:01:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:02::","synset_id":"local.a.03"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:01::","synset_id":"local.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":220,"reasoning_output_tokens":204},"cost":{"currency":"USD","total_usd":0.009080000000000001,"input_uncached_usd":0.00248,"input_cached_usd":0.0,"output_usd":0.0066,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.907652624999173}
{"item_id":"senseval3.d001.s031.t003","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":3,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":4,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s031.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":644,"cached_input_tokens":0,"output_tokens":175,"reasoning_output_tokens":160},"cost":{"currency":"USD","total_usd":0.00847,"input_uncached_usd":0.00322,"input_cached_usd":0.0,"output_usd":0.00525,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9376931249862537}
{"item_id":"senseval3.d001.s031.t004","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"},{"index":3,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval3.d001.s031.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":507,"cached_input_tokens":0,"output_tokens":402,"reasoning_output_tokens":386},"cost":{"currency":"USD","total_usd":0.014595,"input_uncached_usd":0.0025350000000000004,"input_cached_usd":0.0,"output_usd":0.01206,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.183958832989447}
{"item_id":"senseval3.d001.s031.t005","gold_sense_keys":["making%1:04:00::"],"candidates":[{"index":1,"sense_key":"making%1:06:00::","synset_id":"making.n.03"},{"index":2,"sense_key":"making%1:07:00::","synset_id":"qualification.n.01"},{"index":3,"sense_key":"making%1:04:00::","synset_id":"devising.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"making%1:04:00::","call_ids":["senseval3.d001.s031.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"making%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.00489,"input_uncached_usd":0.0026400000000000004,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7461875409353524}
{"item_id":"senseval3.d001.s031.t007","gold_sense_keys":["chamber%1:14:00::"],"candidates":[{"index":1,"sense_key":"chamber%1:06:00::","synset_id":"chamber.n.01"},{"index":2,"sense_key":"chamber%1:08:00::","synset_id":"chamber.n.02"},{"index":3,"sense_key":"chamber%1:14:00::","synset_id":"chamber.n.04"},{"index":4,"sense_key":"chamber%1:06:01::","synset_id":"bedroom.n.01"},{"index":5,"sense_key":"chamber%1:06:02::","synset_id":"chamber.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"chamber%1:14:00::","call_ids":["senseval3.d001.s031.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"chamber%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.00429,"input_uncached_usd":0.00267,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8233859169995412}
{"item_id":"senseval3.d001.s031.t009","gold_sense_keys":["expect%2:31:00::"],"candidates":[{"index":1,"sense_key":"expect%2:31:00::","synset_id":"expect.v.01"},{"index":2,"sense_key":"expect%2:29:00::","synset_id":"have_a_bun_in_the_oven.v.01"},{"index":3,"sense_key":"expect%2:29:02::","synset_id":"expect.v.05"},{"index":4,"sense_key":"expect%2:31:05::","synset_id":"expect.v.04"},{"index":5,"sense_key":"expect%2:31:01::","synset_id":"expect.v.03"},{"index":6,"sense_key":"expect%2:32:00::","synset_id":"ask.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"expect%2:31:00::","call_ids":["senseval3.d001.s031.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"expect%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.005350000000000001,"input_uncached_usd":0.00319,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7931738749612123}
{"item_id":"senseval3.d001.s031.t010","gold_sense_keys":["republican%1:18:01::"],"candidates":[{"index":1,"sense_key":"republican%1:18:01::","synset_id":"republican.n.01"},{"index":2,"sense_key":"republican%1:18:00::","synset_id":"republican.n.02"},{"index":3,"sense_key":"republican%1:17:00::","synset_id":"republican.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%1:18:01::","call_ids":["senseval3.d001.s031.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.004070000000000001,"input_uncached_usd":0.00236,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.429272250039503}
{"item_id":"senseval3.d001.s031.t011","gold_sense_keys":["willing%3:00:00::"],"candidates":[{"index":1,"sense_key":"willing%5:00:00:voluntary:01","synset_id":"uncoerced.s.01"},{"index":2,"sense_key":"willing%3:00:00::","synset_id":"willing.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"willing%3:00:00::","call_ids":["senseval3.d001.s031.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"willing%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.004345,"input_uncached_usd":0.002275,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7342454999452457}
{"item_id":"senseval3.d001.s031.t012","gold_sense_keys":["engage%2:41:06::"],"candidates":[{"index":1,"sense_key":"engage%2:41:01::","synset_id":"hire.v.01"},{"index":2,"sense_key":"engage%2:31:00::","synset_id":"absorb.v.09"},{"index":3,"sense_key":"engage%2:35:00::","synset_id":"engage.v.10"},{"index":4,"sense_key":"engage%2:41:00::","synset_id":"engage.v.04"},{"index":5,"sense_key":"engage%2:32:00::","synset_id":"betroth.v.01"},{"index":6,"sense_key":"engage%2:41:12::","synset_id":"engage.v.07"},{"index":7,"sense_key":"engage%2:41:06::","synset_id":"prosecute.v.03"},{"index":8,"sense_key":"engage%2:40:01::","synset_id":"engage.v.08"},{"index":9,"sense_key":"engage%2:40:00::","synset_id":"lease.v.04"},{"index":10,"sense_key":"engage%2:30:00::","synset_id":"engage.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"engage%2:41:06::","call_ids":["senseval3.d001.s031.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"engage%2:41:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":790,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.006050000000000001,"input_uncached_usd":0.00395,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8809024170041084}
{"item_id":"senseval3.d001.s031.t013","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:00:02::","synset_id":"local.a.03"},{"index":3,"sense_key":"local%3:01:01::","synset_id":"local.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s031.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":261,"reasoning_output_tokens":245},"cost":{"currency":"USD","total_usd":0.01031,"input_uncached_usd":0.00248,"input_cached_usd":0.0,"output_usd":0.00783,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.965761750005186}
{"item_id":"senseval3.d001.s031.t015","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s031.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004375,"input_uncached_usd":0.002575,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5788112910231575}
{"item_id":"senseval3.d001.s031.t016","gold_sense_keys":["counterpart%1:09:00::"],"candidates":[{"index":1,"sense_key":"counterpart%1:06:00::","synset_id":"counterpart.n.02"},{"index":2,"sense_key":"counterpart%1:09:00::","synset_id":"counterpart.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"counterpart%1:09:00::","call_ids":["senseval3.d001.s031.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"counterpart%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.00444,"input_uncached_usd":0.00222,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7878385829972103}
{"item_id":"senseval3.d001.s032.t000","gold_sense_keys":["empirical%3:00:00::"],"candidates":[{"index":1,"sense_key":"empirical%3:00:00::","synset_id":"empirical.a.01"},{"index":2,"sense_key":"empirical%3:01:00::","synset_id":"empiric.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"empirical%3:00:00::","call_ids":["senseval3.d001.s032.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"empirical%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.004135,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9990191670367494}
{"item_id":"senseval3.d001.s032.t001","gold_sense_keys":["support%1:04:02::"],"candidates":[{"index":1,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":2,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":3,"sense_key":"support%1:21:01::","synset_id":"support.n.11"},{"index":4,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":5,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":6,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":7,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":8,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":9,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":10,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":11,"sense_key":"support%1:21:00::","synset_id":"support.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"support%1:04:02::","call_ids":["senseval3.d001.s032.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"support%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":891,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":95},"cost":{"currency":"USD","total_usd":0.007785,"input_uncached_usd":0.004455000000000001,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":120.24396354192868}
{"item_id":"senseval3.d001.s033.t000","gold_sense_keys":["piece%1:06:00::"],"candidates":[{"index":1,"sense_key":"piece%1:13:00::","synset_id":"piece.n.08"},{"index":2,"sense_key":"piece%1:06:04::","synset_id":"objet_d'art.n.01"},{"index":3,"sense_key":"piece%1:06:02::","synset_id":"man.n.10"},{"index":4,"sense_key":"piece%1:06:05::","synset_id":"piece.n.02"},{"index":5,"sense_key":"piece%1:28:00::","synset_id":"while.n.01"},{"index":6,"sense_key":"piece%1:11:00::","synset_id":"piece.n.05"},{"index":7,"sense_key":"piece%1:07:00::","synset_id":"piece.n.09"},{"index":8,"sense_key":"piece%1:06:03::","synset_id":"firearm.n.01"},{"index":9,"sense_key":"piece%1:10:02::","synset_id":"piece.n.06"},{"index":10,"sense_key":"piece%1:21:00::","synset_id":"slice.n.01"},{"index":11,"sense_key":"piece%1:17:00::","synset_id":"part.n.03"},{"index":12,"sense_key":"piece%1:06:00::","synset_id":"piece.n.01"},{"index":13,"sense_key":"piece%1:10:01::","synset_id":"musical_composition.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"piece%1:06:00::","call_ids":["senseval3.d001.s033.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"piece%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":927,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.008145,"input_uncached_usd":0.004635,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2840647499542683}
{"item_id":"senseval3.d001.s033.t001","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":2,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"},{"index":3,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval3.d001.s033.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.004275,"input_uncached_usd":0.002445,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6234873339999467}
{"item_id":"senseval3.d001.s033.t004","gold_sense_keys":["assumption%1:09:00::","assumption%1:10:00::"],"candidates":[{"index":1,"sense_key":"assumption%1:07:00::","synset_id":"presumption.n.03"},{"index":2,"sense_key":"assumption%1:28:00::","synset_id":"assumption.n.04"},{"index":3,"sense_key":"assumption%1:09:00::","synset_id":"assumption.n.02"},{"index":4,"sense_key":"assumption%1:10:00::","synset_id":"premise.n.01"},{"index":5,"sense_key":"assumption%1:04:00::","synset_id":"assumption.n.03"},{"index":6,"sense_key":"assumption%1:04:01::","synset_id":"assumption.n.07"},{"index":7,"sense_key":"assumption%1:11:00::","synset_id":"assumption.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"assumption%1:09:00::","call_ids":["senseval3.d001.s033.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"assumption%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":4307,"reasoning_output_tokens":4241},"cost":{"currency":"USD","total_usd":0.132835,"input_uncached_usd":0.003625,"input_cached_usd":0.0,"output_usd":0.12921,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":60.53229633299634}
{"item_id":"senseval3.d001.s033.t005","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":2,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s033.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004325000000000001,"input_uncached_usd":0.0025250000000000003,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6330963749205694}
{"item_id":"senseval3.d001.s033.t007","gold_sense_keys":["willing%3:00:00::"],"candidates":[{"index":1,"sense_key":"willing%5:00:00:voluntary:01","synset_id":"uncoerced.s.01"},{"index":2,"sense_key":"willing%3:00:00::","synset_id":"willing.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"willing%3:00:00::","call_ids":["senseval3.d001.s033.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"willing%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.004115,"input_uncached_usd":0.002225,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1732959169894457}
{"item_id":"senseval3.d001.s033.t008","gold_sense_keys":["engage%2:41:06::"],"candidates":[{"index":1,"sense_key":"engage%2:41:00::","synset_id":"engage.v.04"},{"index":2,"sense_key":"engage%2:30:00::","synset_id":"engage.v.06"},{"index":3,"sense_key":"engage%2:41:12::","synset_id":"engage.v.07"},{"index":4,"sense_key":"engage%2:32:00::","synset_id":"betroth.v.01"},{"index":5,"sense_key":"engage%2:35:00::","synset_id":"engage.v.10"},{"index":6,"sense_key":"engage%2:40:01::","synset_id":"engage.v.08"},{"index":7,"sense_key":"engage%2:40:00::","synset_id":"lease.v.04"},{"index":8,"sense_key":"engage%2:31:00::","synset_id":"absorb.v.09"},{"index":9,"sense_key":"engage%2:41:01::","synset_id":"hire.v.01"},{"index":10,"sense_key":"engage%2:41:06::","synset_id":"prosecute.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"engage%2:41:06::","call_ids":["senseval3.d001.s033.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"engage%2:41:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":780,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.00543,"input_uncached_usd":0.0039000000000000003,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4139630839927122}
{"item_id":"senseval3.d001.s033.t009","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":2,"sense_key":"local%3:00:02::","synset_id":"local.a.03"},{"index":3,"sense_key":"local%3:00:01::","synset_id":"local.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s033.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":359,"reasoning_output_tokens":343},"cost":{"currency":"USD","total_usd":0.0132,"input_uncached_usd":0.0024300000000000003,"input_cached_usd":0.0,"output_usd":0.01077,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.681180790998042}
{"item_id":"senseval3.d001.s033.t011","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s033.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.004135,"input_uncached_usd":0.002455,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.03615279099904}
{"item_id":"senseval3.d001.s033.t012","gold_sense_keys":["colleague%1:18:01::"],"candidates":[{"index":1,"sense_key":"colleague%1:18:01::","synset_id":"colleague.n.02"},{"index":2,"sense_key":"colleague%1:18:00::","synset_id":"colleague.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"colleague%1:18:01::","call_ids":["senseval3.d001.s033.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"colleague%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":271,"reasoning_output_tokens":255},"cost":{"currency":"USD","total_usd":0.01041,"input_uncached_usd":0.0022800000000000003,"input_cached_usd":0.0,"output_usd":0.00813,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.957640999928117}
{"item_id":"senseval3.d001.s034.t000","gold_sense_keys":["first%4:02:00::"],"candidates":[{"index":1,"sense_key":"first%4:02:03::","synset_id":"first.r.03"},{"index":2,"sense_key":"first%4:02:02::","synset_id":"first.r.02"},{"index":3,"sense_key":"first%4:02:01::","synset_id":"foremost.r.01"},{"index":4,"sense_key":"first%4:02:00::","synset_id":"first.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"first%4:02:00::","call_ids":["senseval3.d001.s034.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"first%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.00482,"input_uncached_usd":0.00266,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9752322089625522}
{"item_id":"senseval3.d001.s034.t002","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":2,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":3,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":4,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":5,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":6,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":7,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":8,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":9,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":10,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":11,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":12,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"},{"index":13,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":14,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"find%2:32:00::","call_ids":["senseval3.d001.s034.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"find%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1168,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02168,"input_uncached_usd":0.005840000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.188755374983884}
{"item_id":"senseval3.d001.s034.t006","gold_sense_keys":["personal%3:00:00::","personal%5:00:00:subjective:00"],"candidates":[{"index":1,"sense_key":"personal%3:01:01::","synset_id":"personal.a.03"},{"index":2,"sense_key":"personal%5:00:00:physical:00","synset_id":"personal.s.02"},{"index":3,"sense_key":"personal%3:00:00::","synset_id":"personal.a.01"},{"index":4,"sense_key":"personal%5:00:00:subjective:00","synset_id":"personal.s.01"},{"index":5,"sense_key":"personal%3:01:00::","synset_id":"personal.a.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"personal%3:00:00::","call_ids":["senseval3.d001.s034.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"personal%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018505,"input_uncached_usd":0.0026650000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.296402334002778}
{"item_id":"senseval3.d001.s034.t007","gold_sense_keys":["staff%1:14:01::"],"candidates":[{"index":1,"sense_key":"staff%1:06:00::","synset_id":"staff.n.02"},{"index":2,"sense_key":"staff%1:27:00::","synset_id":"staff.n.04"},{"index":3,"sense_key":"staff%1:10:01::","synset_id":"staff.n.05"},{"index":4,"sense_key":"staff%1:14:00::","synset_id":"staff.n.03"},{"index":5,"sense_key":"staff%1:14:01::","synset_id":"staff.n.01"},{"index":6,"sense_key":"staff%1:10:00::","synset_id":"staff.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"staff%1:14:01::","call_ids":["senseval3.d001.s034.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"staff%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.005095000000000001,"input_uncached_usd":0.0030550000000000004,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7847807919606566}
{"item_id":"senseval3.d001.s034.t008","gold_sense_keys":["budget%1:21:02::"],"candidates":[{"index":1,"sense_key":"budget%1:21:03::","synset_id":"budget.n.02"},{"index":2,"sense_key":"budget%1:21:02::","synset_id":"budget.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"budget%1:21:02::","call_ids":["senseval3.d001.s034.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"budget%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.00453,"input_uncached_usd":0.0022800000000000003,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.752229665988125}
{"item_id":"senseval3.d001.s034.t009","gold_sense_keys":["democrat%1:18:01::"],"candidates":[{"index":1,"sense_key":"democrat%1:18:00::","synset_id":"democrat.n.02"},{"index":2,"sense_key":"democrat%1:18:01::","synset_id":"democrat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democrat%1:18:01::","call_ids":["senseval3.d001.s034.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democrat%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":429,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.004215,"input_uncached_usd":0.002145,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0884978329995647}
{"item_id":"senseval3.d001.s035.t000","gold_sense_keys":["extent%1:26:00::"],"candidates":[{"index":1,"sense_key":"extent%1:26:00::","synset_id":"extent.n.01"},{"index":2,"sense_key":"extent%1:07:00::","synset_id":"extent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"extent%1:26:00::","call_ids":["senseval3.d001.s035.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"extent%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.0041849999999999995,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8594980409834534}
{"item_id":"senseval3.d001.s035.t001","gold_sense_keys":["primary%3:00:00::","primary%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"primary%5:00:00:important:00","synset_id":"chief.s.01"},{"index":2,"sense_key":"primary%5:00:00:underived:00","synset_id":"primary.s.01"},{"index":3,"sense_key":"primary%3:00:00::","synset_id":"primary.a.01"},{"index":4,"sense_key":"primary%5:00:00:basic:00","synset_id":"elementary.s.02"},{"index":5,"sense_key":"primary%5:00:00:essential:00","synset_id":"basal.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"primary%5:00:00:important:00","call_ids":["senseval3.d001.s035.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"primary%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":577,"reasoning_output_tokens":555},"cost":{"currency":"USD","total_usd":0.020160000000000004,"input_uncached_usd":0.00285,"input_cached_usd":0.0,"output_usd":0.01731,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.919365375069901}
{"item_id":"senseval3.d001.s035.t002","gold_sense_keys":["duty%1:04:02::"],"candidates":[{"index":1,"sense_key":"duty%1:04:00::","synset_id":"duty.n.01"},{"index":2,"sense_key":"duty%1:04:02::","synset_id":"duty.n.02"},{"index":3,"sense_key":"duty%1:21:00::","synset_id":"duty.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"duty%1:04:02::","call_ids":["senseval3.d001.s035.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"duty%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.004605,"input_uncached_usd":0.0026550000000000002,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2557792499428615}
{"item_id":"senseval3.d001.s035.t003","gold_sense_keys":["personal%3:00:00::","personal%5:00:00:subjective:00"],"candidates":[{"index":1,"sense_key":"personal%3:01:01::","synset_id":"personal.a.03"},{"index":2,"sense_key":"personal%3:00:00::","synset_id":"personal.a.01"},{"index":3,"sense_key":"personal%3:01:00::","synset_id":"personal.a.05"},{"index":4,"sense_key":"personal%5:00:00:physical:00","synset_id":"personal.s.02"},{"index":5,"sense_key":"personal%5:00:00:subjective:00","synset_id":"personal.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"personal%5:00:00:subjective:00","call_ids":["senseval3.d001.s035.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"personal%5:00:00:subjective:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":1045,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.033990000000000006,"input_uncached_usd":0.0026400000000000004,"input_cached_usd":0.0,"output_usd":0.03135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":15.708679375005886}
{"item_id":"senseval3.d001.s035.t004","gold_sense_keys":["staff%1:14:01::"],"candidates":[{"index":1,"sense_key":"staff%1:14:00::","synset_id":"staff.n.03"},{"index":2,"sense_key":"staff%1:10:00::","synset_id":"staff.n.06"},{"index":3,"sense_key":"staff%1:14:01::","synset_id":"staff.n.01"},{"index":4,"sense_key":"staff%1:27:00::","synset_id":"staff.n.04"},{"index":5,"sense_key":"staff%1:10:01::","synset_id":"staff.n.05"},{"index":6,"sense_key":"staff%1:06:00::","synset_id":"staff.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"staff%1:14:01::","call_ids":["senseval3.d001.s035.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"staff%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.00456,"input_uncached_usd":0.00303,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.637518292060122}
{"item_id":"senseval3.d001.s035.t005","gold_sense_keys":["involve%2:42:00::"],"candidates":[{"index":1,"sense_key":"involve%2:42:07::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"involve%2:42:04::","synset_id":"involve.v.02"},{"index":3,"sense_key":"involve%2:42:02::","synset_id":"involve.v.05"},{"index":4,"sense_key":"involve%2:31:00::","synset_id":"involve.v.06"},{"index":5,"sense_key":"involve%2:30:00::","synset_id":"involve.v.07"},{"index":6,"sense_key":"involve%2:42:01::","synset_id":"involve.v.01"},{"index":7,"sense_key":"involve%2:42:00::","synset_id":"imply.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"involve%2:42:02::","call_ids":["senseval3.d001.s035.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"involve%2:42:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019035,"input_uncached_usd":0.0031950000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.332733457908034}
{"item_id":"senseval3.d001.s035.t006","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:02::","synset_id":"local.a.03"},{"index":2,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":3,"sense_key":"local%3:01:01::","synset_id":"local.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:00:01::","call_ids":["senseval3.d001.s035.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01829,"input_uncached_usd":0.0024500000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.226590499980375}
{"item_id":"senseval3.d001.s035.t008","gold_sense_keys":["indicate%2:32:01::","indicate%2:32:02::"],"candidates":[{"index":1,"sense_key":"indicate%2:32:01::","synset_id":"bespeak.v.01"},{"index":2,"sense_key":"indicate%2:32:00::","synset_id":"indicate.v.03"},{"index":3,"sense_key":"indicate%2:32:04::","synset_id":"indicate.v.05"},{"index":4,"sense_key":"indicate%2:32:03::","synset_id":"indicate.v.02"},{"index":5,"sense_key":"indicate%2:32:02::","synset_id":"argue.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"indicate%2:32:02::","call_ids":["senseval3.d001.s035.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"indicate%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":147},"cost":{"currency":"USD","total_usd":0.007905,"input_uncached_usd":0.0030450000000000004,"input_cached_usd":0.0,"output_usd":0.00486,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.969450750038959}
{"item_id":"senseval3.d001.s035.t009","gold_sense_keys":["political%3:01:01::"],"candidates":[{"index":1,"sense_key":"political%3:01:02::","synset_id":"political.a.03"},{"index":2,"sense_key":"political%3:01:01::","synset_id":"political.a.02"},{"index":3,"sense_key":"political%3:00:00::","synset_id":"political.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"political%3:01:01::","call_ids":["senseval3.d001.s035.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"political%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.004575,"input_uncached_usd":0.002445,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.890257874969393}
{"item_id":"senseval3.d001.s035.t010","gold_sense_keys":["philosophy%1:09:01::"],"candidates":[{"index":1,"sense_key":"philosophy%1:09:02::","synset_id":"philosophy.n.03"},{"index":2,"sense_key":"philosophy%1:09:01::","synset_id":"doctrine.n.01"},{"index":3,"sense_key":"philosophy%1:09:00::","synset_id":"philosophy.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"philosophy%1:09:01::","call_ids":["senseval3.d001.s035.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"philosophy%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.00551,"input_uncached_usd":0.00254,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8261252080556005}
{"item_id":"senseval3.d001.s035.t011","gold_sense_keys":["lead%2:32:00::"],"candidates":[{"index":1,"sense_key":"lead%2:41:12::","synset_id":"contribute.v.03"},{"index":2,"sense_key":"lead%2:42:00::","synset_id":"run.v.03"},{"index":3,"sense_key":"lead%2:42:01::","synset_id":"lead.v.08"},{"index":4,"sense_key":"lead%2:42:02::","synset_id":"run.v.23"},{"index":5,"sense_key":"lead%2:38:00::","synset_id":"lead.v.04"},{"index":6,"sense_key":"lead%2:42:03::","synset_id":"go.v.25"},{"index":7,"sense_key":"lead%2:42:12::","synset_id":"lead.v.03"},{"index":8,"sense_key":"lead%2:38:02::","synset_id":"precede.v.04"},{"index":9,"sense_key":"lead%2:32:00::","synset_id":"lead.v.05"},{"index":10,"sense_key":"lead%2:42:04::","synset_id":"leave.v.07"},{"index":11,"sense_key":"lead%2:41:00::","synset_id":"head.v.02"},{"index":12,"sense_key":"lead%2:38:01::","synset_id":"lead.v.01"},{"index":13,"sense_key":"lead%2:36:00::","synset_id":"conduct.v.02"},{"index":14,"sense_key":"lead%2:32:01::","synset_id":"moderate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"lead%2:32:00::","call_ids":["senseval3.d001.s035.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"lead%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":953,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":119},"cost":{"currency":"USD","total_usd":0.008815,"input_uncached_usd":0.004765,"input_cached_usd":0.0,"output_usd":0.00405,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.173711708048359}
{"item_id":"senseval3.d001.s035.t013","gold_sense_keys":["republican%1:18:01::"],"candidates":[{"index":1,"sense_key":"republican%1:17:00::","synset_id":"republican.n.03"},{"index":2,"sense_key":"republican%1:18:00::","synset_id":"republican.n.02"},{"index":3,"sense_key":"republican%1:18:01::","synset_id":"republican.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"republican%1:18:01::","call_ids":["senseval3.d001.s035.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"republican%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":466,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.00368,"input_uncached_usd":0.00233,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.415553625090979}
{"item_id":"senseval3.d001.s035.t014","gold_sense_keys":["narrow%5:00:00:limited:00"],"candidates":[{"index":1,"sense_key":"narrow%3:00:02::","synset_id":"narrow.a.04"},{"index":2,"sense_key":"narrow%3:00:00::","synset_id":"narrow.a.01"},{"index":3,"sense_key":"narrow%5:00:00:careful:00","synset_id":"minute.s.02"},{"index":4,"sense_key":"narrow%3:00:06::","synset_id":"narrow-minded.a.02"},{"index":5,"sense_key":"narrow%5:00:00:limited:00","synset_id":"narrow.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"narrow%5:00:00:limited:00","call_ids":["senseval3.d001.s035.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"narrow%5:00:00:limited:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":166,"reasoning_output_tokens":150},"cost":{"currency":"USD","total_usd":0.007760000000000001,"input_uncached_usd":0.0027800000000000004,"input_cached_usd":0.0,"output_usd":0.00498,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.472491207998246}
{"item_id":"senseval3.d001.s035.t015","gold_sense_keys":["constituent%1:18:00::"],"candidates":[{"index":1,"sense_key":"constituent%1:18:00::","synset_id":"constituent.n.02"},{"index":2,"sense_key":"constituent%1:09:00::","synset_id":"component.n.01"},{"index":3,"sense_key":"constituent%1:24:00::","synset_id":"part.n.01"},{"index":4,"sense_key":"constituent%1:10:00::","synset_id":"constituent.n.04"},{"index":5,"sense_key":"constituent%1:06:00::","synset_id":"component.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"constituent%1:18:00::","call_ids":["senseval3.d001.s035.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"constituent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":98},"cost":{"currency":"USD","total_usd":0.006710000000000001,"input_uncached_usd":0.0032900000000000004,"input_cached_usd":0.0,"output_usd":0.0034200000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.565431416966021}
{"item_id":"senseval3.d001.s035.t016","gold_sense_keys":["concern%1:09:00::"],"candidates":[{"index":1,"sense_key":"concern%1:12:01::","synset_id":"concern.n.02"},{"index":2,"sense_key":"concern%1:09:00::","synset_id":"concern.n.01"},{"index":3,"sense_key":"concern%1:12:00::","synset_id":"concern.n.03"},{"index":4,"sense_key":"concern%1:09:01::","synset_id":"concern.n.04"},{"index":5,"sense_key":"concern%1:14:00::","synset_id":"business.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"concern%1:09:00::","call_ids":["senseval3.d001.s035.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"concern%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.006,"input_uncached_usd":0.00303,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.577611165936105}
{"item_id":"senseval3.d001.s036.t002","gold_sense_keys":["assumption%1:09:00::","assumption%1:10:00::"],"candidates":[{"index":1,"sense_key":"assumption%1:04:00::","synset_id":"assumption.n.03"},{"index":2,"sense_key":"assumption%1:11:00::","synset_id":"assumption.n.05"},{"index":3,"sense_key":"assumption%1:07:00::","synset_id":"presumption.n.03"},{"index":4,"sense_key":"assumption%1:09:00::","synset_id":"assumption.n.02"},{"index":5,"sense_key":"assumption%1:10:00::","synset_id":"premise.n.01"},{"index":6,"sense_key":"assumption%1:04:01::","synset_id":"assumption.n.07"},{"index":7,"sense_key":"assumption%1:28:00::","synset_id":"assumption.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"assumption%1:10:00::","call_ids":["senseval3.d001.s036.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"assumption%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019455,"input_uncached_usd":0.003615,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.779161833925173}
{"item_id":"senseval3.d001.s036.t003","gold_sense_keys":["valid%3:00:00::"],"candidates":[{"index":1,"sense_key":"valid%3:00:00::","synset_id":"valid.a.01"},{"index":2,"sense_key":"valid%5:00:00:unexpired:00","synset_id":"valid.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"valid%3:00:00::","call_ids":["senseval3.d001.s036.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"valid%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.0039000000000000003,"input_uncached_usd":0.00216,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.423758207936771}
{"item_id":"senseval3.d001.s036.t004","gold_sense_keys":["democrat%1:18:01::"],"candidates":[{"index":1,"sense_key":"democrat%1:18:00::","synset_id":"democrat.n.02"},{"index":2,"sense_key":"democrat%1:18:01::","synset_id":"democrat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democrat%1:18:01::","call_ids":["senseval3.d001.s036.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democrat%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.004070000000000001,"input_uncached_usd":0.0020900000000000003,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9857958330539986}
{"item_id":"senseval3.d001.s036.t005","gold_sense_keys":["have%2:30:01::","have%2:40:00::","have%2:42:00::","have%2:42:12::"],"candidates":[{"index":1,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":2,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":3,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":4,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":5,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":6,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":7,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":8,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":9,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":10,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":11,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":12,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":13,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":14,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":15,"sense_key":"have%2:35:00::","synset_id":"take.v.35"},{"index":16,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":17,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":18,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:42:00::","call_ids":["senseval3.d001.s036.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1201,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.021815,"input_uncached_usd":0.006005000000000001,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.59982524998486}
{"item_id":"senseval3.d001.s036.t006","gold_sense_keys":["attendance%1:04:00::"],"candidates":[{"index":1,"sense_key":"attendance%1:04:00::","synset_id":"attendance.n.01"},{"index":2,"sense_key":"attendance%1:14:00::","synset_id":"attendance.n.03"},{"index":3,"sense_key":"attendance%1:28:00::","synset_id":"attendance.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"attendance%1:28:00::","call_ids":["senseval3.d001.s036.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"attendance%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.00464,"input_uncached_usd":0.00236,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9325252079870552}
{"item_id":"senseval3.d001.s036.t007","gold_sense_keys":["rate%1:24:00::"],"candidates":[{"index":1,"sense_key":"rate%1:24:00::","synset_id":"rate.n.04"},{"index":2,"sense_key":"rate%1:21:00::","synset_id":"rate.n.02"},{"index":3,"sense_key":"rate%1:07:00::","synset_id":"pace.n.03"},{"index":4,"sense_key":"rate%1:28:00::","synset_id":"rate.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rate%1:24:00::","call_ids":["senseval3.d001.s036.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rate%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.0052,"input_uncached_usd":0.0027400000000000002,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.158283125027083}
{"item_id":"senseval3.d001.s036.t009","gold_sense_keys":["vote%1:04:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":2,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":3,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"},{"index":4,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":5,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"vote%1:04:00::","call_ids":["senseval3.d001.s036.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"vote%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":348,"reasoning_output_tokens":332},"cost":{"currency":"USD","total_usd":0.013434999999999999,"input_uncached_usd":0.0029950000000000003,"input_cached_usd":0.0,"output_usd":0.01044,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.492129208985716}
{"item_id":"senseval3.d001.s036.t010","gold_sense_keys":["republican%1:18:01::"],"candidates":[{"index":1,"sense_key":"republican%1:18:00::","synset_id":"republican.n.02"},{"index":2,"sense_key":"republican%1:17:00::","synset_id":"republican.n.03"},{"index":3,"sense_key":"republican%1:18:01::","synset_id":"republican.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"republican%1:18:01::","call_ids":["senseval3.d001.s036.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"republican%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.0038900000000000002,"input_uncached_usd":0.0023000000000000004,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5628018750576302}
{"item_id":"senseval3.d001.s036.t011","gold_sense_keys":["extent%1:26:00::"],"candidates":[{"index":1,"sense_key":"extent%1:07:00::","synset_id":"extent.n.02"},{"index":2,"sense_key":"extent%1:26:00::","synset_id":"extent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"extent%1:26:00::","call_ids":["senseval3.d001.s036.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"extent%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004215,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.967371082981117}
{"item_id":"senseval3.d001.s036.t013","gold_sense_keys":["vote%1:04:00::"],"candidates":[{"index":1,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":2,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":3,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":4,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":5,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"vote%1:04:00::","call_ids":["senseval3.d001.s036.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"vote%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":478,"reasoning_output_tokens":462},"cost":{"currency":"USD","total_usd":0.017335,"input_uncached_usd":0.0029950000000000003,"input_cached_usd":0.0,"output_usd":0.01434,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.348550333059393}
{"item_id":"senseval3.d001.s036.t014","gold_sense_keys":["reflect%2:39:01::"],"candidates":[{"index":1,"sense_key":"reflect%2:32:01::","synset_id":"reflect.v.06"},{"index":2,"sense_key":"reflect%2:39:00::","synset_id":"reflect.v.03"},{"index":3,"sense_key":"reflect%2:43:00::","synset_id":"reflect.v.04"},{"index":4,"sense_key":"reflect%2:39:01::","synset_id":"reflect.v.01"},{"index":5,"sense_key":"reflect%2:32:00::","synset_id":"reflect.v.07"},{"index":6,"sense_key":"reflect%2:39:02::","synset_id":"reflect.v.05"},{"index":7,"sense_key":"reflect%2:31:00::","synset_id":"chew_over.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"reflect%2:39:01::","call_ids":["senseval3.d001.s036.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"reflect%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":137},"cost":{"currency":"USD","total_usd":0.007785,"input_uncached_usd":0.0032250000000000004,"input_cached_usd":0.0,"output_usd":0.00456,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0778432500083}
{"item_id":"senseval3.d001.s036.t015","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":2,"sense_key":"national%3:01:01::","synset_id":"national.a.07"},{"index":3,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":4,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":5,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s036.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":632,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":115},"cost":{"currency":"USD","total_usd":0.007090000000000001,"input_uncached_usd":0.00316,"input_cached_usd":0.0,"output_usd":0.00393,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2742445410694927}
{"item_id":"senseval3.d001.s036.t016","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":2,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"},{"index":3,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval3.d001.s036.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":377,"reasoning_output_tokens":362},"cost":{"currency":"USD","total_usd":0.013785,"input_uncached_usd":0.002475,"input_cached_usd":0.0,"output_usd":0.01131,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.9839904580730945}
{"item_id":"senseval3.d001.s036.t017","gold_sense_keys":["making%1:04:00::"],"candidates":[{"index":1,"sense_key":"making%1:07:00::","synset_id":"qualification.n.01"},{"index":2,"sense_key":"making%1:06:00::","synset_id":"making.n.03"},{"index":3,"sense_key":"making%1:04:00::","synset_id":"devising.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"making%1:04:00::","call_ids":["senseval3.d001.s036.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"making%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.0045000000000000005,"input_uncached_usd":0.0025800000000000003,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.472056958009489}
{"item_id":"senseval3.d001.s036.t020","gold_sense_keys":["vote%1:04:00::"],"candidates":[{"index":1,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":2,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"},{"index":3,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":4,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":5,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"vote%1:04:00::","call_ids":["senseval3.d001.s036.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"vote%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":210,"reasoning_output_tokens":194},"cost":{"currency":"USD","total_usd":0.009295000000000001,"input_uncached_usd":0.0029950000000000003,"input_cached_usd":0.0,"output_usd":0.0063,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.475611000088975}
{"item_id":"senseval3.d001.s036.t021","gold_sense_keys":["time%1:28:05::"],"candidates":[{"index":1,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":2,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":3,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":4,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":5,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"},{"index":6,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":7,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":8,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":9,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":10,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"time%1:28:05::","call_ids":["senseval3.d001.s036.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"time%1:28:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.00589,"input_uncached_usd":0.00385,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2883659170474857}
{"item_id":"senseval3.d001.s036.t023","gold_sense_keys":["devote%2:40:00::"],"candidates":[{"index":1,"sense_key":"devote%2:40:00::","synset_id":"devote.v.03"},{"index":2,"sense_key":"devote%2:31:00::","synset_id":"give.v.10"},{"index":3,"sense_key":"devote%2:32:00::","synset_id":"give.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"devote%2:40:00::","call_ids":["senseval3.d001.s036.t023__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"devote%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":258,"reasoning_output_tokens":242},"cost":{"currency":"USD","total_usd":0.01022,"input_uncached_usd":0.00248,"input_cached_usd":0.0,"output_usd":0.00774,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.990803209017031}
{"item_id":"senseval3.d001.s036.t024","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:02::","synset_id":"local.a.03"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:01::","synset_id":"local.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"local%3:00:01::","call_ids":["senseval3.d001.s036.t024__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"local%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1023},"cost":{"currency":"USD","total_usd":0.033800000000000004,"input_uncached_usd":0.0024200000000000003,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":19.950619625044055}
{"item_id":"senseval3.d001.s037.t001","gold_sense_keys":["indicate%2:32:01::","indicate%2:32:02::"],"candidates":[{"index":1,"sense_key":"indicate%2:32:04::","synset_id":"indicate.v.05"},{"index":2,"sense_key":"indicate%2:32:02::","synset_id":"argue.v.03"},{"index":3,"sense_key":"indicate%2:32:00::","synset_id":"indicate.v.03"},{"index":4,"sense_key":"indicate%2:32:01::","synset_id":"bespeak.v.01"},{"index":5,"sense_key":"indicate%2:32:03::","synset_id":"indicate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"indicate%2:32:02::","call_ids":["senseval3.d001.s037.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"indicate%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":575,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.005455,"input_uncached_usd":0.0028750000000000004,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0104195419698954}
{"item_id":"senseval3.d001.s037.t002","gold_sense_keys":["case%1:26:00::"],"candidates":[{"index":1,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":2,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":3,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":4,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":5,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":6,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":7,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":8,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":9,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":10,"sense_key":"case%1:06:01::","synset_id":"case.n.20"},{"index":11,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":12,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":13,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":14,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":15,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":16,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":17,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":20,"sense_key":"case%1:10:01::","synset_id":"case.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"case%1:26:00::","call_ids":["senseval3.d001.s037.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"case%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1200,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02184,"input_uncached_usd":0.006,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.750461749965325}
{"item_id":"senseval3.d001.s038.t000","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":2,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s038.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00434,"input_uncached_usd":0.00254,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6413223339477554}
{"item_id":"senseval3.d001.s038.t001","gold_sense_keys":["attendance%1:04:00::"],"candidates":[{"index":1,"sense_key":"attendance%1:28:00::","synset_id":"attendance.n.02"},{"index":2,"sense_key":"attendance%1:04:00::","synset_id":"attendance.n.01"},{"index":3,"sense_key":"attendance%1:14:00::","synset_id":"attendance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attendance%1:28:00::","call_ids":["senseval3.d001.s038.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attendance%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":138},"cost":{"currency":"USD","total_usd":0.006975,"input_uncached_usd":0.0023850000000000004,"input_cached_usd":0.0,"output_usd":0.00459,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3203662920277566}
{"item_id":"senseval3.d001.s038.t002","gold_sense_keys":["rate%1:24:00::"],"candidates":[{"index":1,"sense_key":"rate%1:24:00::","synset_id":"rate.n.04"},{"index":2,"sense_key":"rate%1:07:00::","synset_id":"pace.n.03"},{"index":3,"sense_key":"rate%1:21:00::","synset_id":"rate.n.02"},{"index":4,"sense_key":"rate%1:28:00::","synset_id":"rate.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rate%1:24:00::","call_ids":["senseval3.d001.s038.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rate%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.005825,"input_uncached_usd":0.002765,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.330979791935533}
{"item_id":"senseval3.d001.s038.t004","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s038.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.0044800000000000005,"input_uncached_usd":0.0024700000000000004,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1931803330080584}
{"item_id":"senseval3.d001.s038.t005","gold_sense_keys":["attendance%1:04:00::"],"candidates":[{"index":1,"sense_key":"attendance%1:04:00::","synset_id":"attendance.n.01"},{"index":2,"sense_key":"attendance%1:28:00::","synset_id":"attendance.n.02"},{"index":3,"sense_key":"attendance%1:14:00::","synset_id":"attendance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"attendance%1:28:00::","call_ids":["senseval3.d001.s038.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"attendance%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.004635,"input_uncached_usd":0.0023850000000000004,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9304579580202699}
{"item_id":"senseval3.d001.s038.t006","gold_sense_keys":["rate%1:24:00::"],"candidates":[{"index":1,"sense_key":"rate%1:24:00::","synset_id":"rate.n.04"},{"index":2,"sense_key":"rate%1:28:00::","synset_id":"rate.n.01"},{"index":3,"sense_key":"rate%1:07:00::","synset_id":"pace.n.03"},{"index":4,"sense_key":"rate%1:21:00::","synset_id":"rate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rate%1:24:00::","call_ids":["senseval3.d001.s038.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rate%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.005375,"input_uncached_usd":0.002765,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.365173916099593}
{"item_id":"senseval3.d001.s039.t000","gold_sense_keys":["show%2:32:00::","show%2:32:03::","show%2:36:00::"],"candidates":[{"index":1,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":2,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":5,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":6,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":7,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":8,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":9,"sense_key":"show%2:33:00::","synset_id":"show.v.12"},{"index":10,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":11,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":12,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"show%2:32:03::","call_ids":["senseval3.d001.s039.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"show%2:32:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":988,"cached_input_tokens":0,"output_tokens":353,"reasoning_output_tokens":338},"cost":{"currency":"USD","total_usd":0.015530000000000002,"input_uncached_usd":0.004940000000000001,"input_cached_usd":0.0,"output_usd":0.01059,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.039892500033602}
{"item_id":"senseval3.d001.s039.t001","gold_sense_keys":["table%1:14:00::"],"candidates":[{"index":1,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":2,"sense_key":"table%1:14:01::","synset_id":"table.n.05"},{"index":3,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":4,"sense_key":"table%1:13:01::","synset_id":"board.n.04"},{"index":5,"sense_key":"table%1:14:00::","synset_id":"table.n.01"},{"index":6,"sense_key":"table%1:06:02::","synset_id":"table.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"table%1:14:00::","call_ids":["senseval3.d001.s039.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"table%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00513,"input_uncached_usd":0.00333,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8340907919919118}
{"item_id":"senseval3.d001.s039.t002","gold_sense_keys":["democrat%1:18:01::"],"candidates":[{"index":1,"sense_key":"democrat%1:18:01::","synset_id":"democrat.n.01"},{"index":2,"sense_key":"democrat%1:18:00::","synset_id":"democrat.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"democrat%1:18:01::","call_ids":["senseval3.d001.s039.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"democrat%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.004485,"input_uncached_usd":0.0023550000000000003,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.993002416100353}
{"item_id":"senseval3.d001.s039.t004","gold_sense_keys":["proportion%1:24:01::"],"candidates":[{"index":1,"sense_key":"proportion%1:07:00::","synset_id":"proportion.n.02"},{"index":2,"sense_key":"proportion%1:07:01::","synset_id":"proportion.n.05"},{"index":3,"sense_key":"proportion%1:24:00::","synset_id":"proportion.n.01"},{"index":4,"sense_key":"proportion%1:25:00::","synset_id":"symmetry.n.02"},{"index":5,"sense_key":"proportion%1:24:01::","synset_id":"proportion.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"proportion%1:24:00::","call_ids":["senseval3.d001.s039.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"proportion%1:24:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":655,"cached_input_tokens":0,"output_tokens":1563,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.050165,"input_uncached_usd":0.003275,"input_cached_usd":0.0,"output_usd":0.04689,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":29.81977254198864}
{"item_id":"senseval3.d001.s039.t005","gold_sense_keys":["personal%3:00:00::","personal%5:00:00:subjective:00"],"candidates":[{"index":1,"sense_key":"personal%3:01:00::","synset_id":"personal.a.05"},{"index":2,"sense_key":"personal%5:00:00:physical:00","synset_id":"personal.s.02"},{"index":3,"sense_key":"personal%3:00:00::","synset_id":"personal.a.01"},{"index":4,"sense_key":"personal%3:01:01::","synset_id":"personal.a.03"},{"index":5,"sense_key":"personal%5:00:00:subjective:00","synset_id":"personal.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"personal%5:00:00:subjective:00","call_ids":["senseval3.d001.s039.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"personal%5:00:00:subjective:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":575,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018715,"input_uncached_usd":0.0028750000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.225362958037294}
{"item_id":"senseval3.d001.s039.t006","gold_sense_keys":["staff%1:14:01::"],"candidates":[{"index":1,"sense_key":"staff%1:10:00::","synset_id":"staff.n.06"},{"index":2,"sense_key":"staff%1:27:00::","synset_id":"staff.n.04"},{"index":3,"sense_key":"staff%1:10:01::","synset_id":"staff.n.05"},{"index":4,"sense_key":"staff%1:14:01::","synset_id":"staff.n.01"},{"index":5,"sense_key":"staff%1:06:00::","synset_id":"staff.n.02"},{"index":6,"sense_key":"staff%1:14:00::","synset_id":"staff.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"staff%1:14:01::","call_ids":["senseval3.d001.s039.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"staff%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.0051,"input_uncached_usd":0.0032700000000000003,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6281575419707224}
{"item_id":"senseval3.d001.s039.t008","gold_sense_keys":["office%1:06:00::"],"candidates":[{"index":1,"sense_key":"office%1:04:02::","synset_id":"position.n.06"},{"index":2,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":3,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":4,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":6,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":7,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"office%1:06:00::","call_ids":["senseval3.d001.s039.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"office%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.006175,"input_uncached_usd":0.0037150000000000004,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9797939580166712}
{"item_id":"senseval3.d001.s039.t009","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:02::","synset_id":"local.a.03"},{"index":2,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":3,"sense_key":"local%3:01:01::","synset_id":"local.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s039.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018525,"input_uncached_usd":0.0026850000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.015793208032846}
{"item_id":"senseval3.d001.s039.t011","gold_sense_keys":["duty%1:04:02::"],"candidates":[{"index":1,"sense_key":"duty%1:04:02::","synset_id":"duty.n.02"},{"index":2,"sense_key":"duty%1:04:00::","synset_id":"duty.n.01"},{"index":3,"sense_key":"duty%1:21:00::","synset_id":"duty.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"duty%1:04:02::","call_ids":["senseval3.d001.s039.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"duty%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.005145,"input_uncached_usd":0.0028950000000000004,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8473159169079736}
{"item_id":"senseval3.d001.s039.t013","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":2,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":3,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":4,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":5,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":6,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s039.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":168,"reasoning_output_tokens":152},"cost":{"currency":"USD","total_usd":0.008465,"input_uncached_usd":0.003425,"input_cached_usd":0.0,"output_usd":0.00504,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3280723750358447}
{"item_id":"senseval3.d001.s039.t014","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval3.d001.s039.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":273,"reasoning_output_tokens":257},"cost":{"currency":"USD","total_usd":0.010929999999999999,"input_uncached_usd":0.0027400000000000002,"input_cached_usd":0.0,"output_usd":0.00819,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.248448166064918}
{"item_id":"senseval3.d001.s039.t015","gold_sense_keys":["making%1:04:00::"],"candidates":[{"index":1,"sense_key":"making%1:04:00::","synset_id":"devising.n.01"},{"index":2,"sense_key":"making%1:07:00::","synset_id":"qualification.n.01"},{"index":3,"sense_key":"making%1:06:00::","synset_id":"making.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"making%1:04:00::","call_ids":["senseval3.d001.s039.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"making%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":86},"cost":{"currency":"USD","total_usd":0.005875,"input_uncached_usd":0.0028450000000000003,"input_cached_usd":0.0,"output_usd":0.00303,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.119884458021261}
{"item_id":"senseval3.d001.s039.t016","gold_sense_keys":["activity%1:04:00::"],"candidates":[{"index":1,"sense_key":"activity%1:22:00::","synset_id":"bodily_process.n.01"},{"index":2,"sense_key":"activity%1:26:01::","synset_id":"activity.n.04"},{"index":3,"sense_key":"activity%1:26:00::","synset_id":"action.n.02"},{"index":4,"sense_key":"activity%1:22:01::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"activity%1:04:00::","synset_id":"activity.n.01"},{"index":6,"sense_key":"activity%1:07:00::","synset_id":"activeness.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"activity%1:04:00::","call_ids":["senseval3.d001.s039.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"activity%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":671,"cached_input_tokens":0,"output_tokens":178,"reasoning_output_tokens":162},"cost":{"currency":"USD","total_usd":0.008695000000000001,"input_uncached_usd":0.0033550000000000003,"input_cached_usd":0.0,"output_usd":0.00534,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.268248250009492}
{"item_id":"senseval3.d001.s039.t018","gold_sense_keys":["relative%5:00:00:proportionate:00"],"candidates":[{"index":1,"sense_key":"relative%5:00:00:proportionate:00","synset_id":"proportional.s.01"},{"index":2,"sense_key":"relative%3:00:00::","synset_id":"relative.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"relative%5:00:00:proportionate:00","call_ids":["senseval3.d001.s039.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"relative%5:00:00:proportionate:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01839,"input_uncached_usd":0.0025800000000000003,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.029657416045666}
{"item_id":"senseval3.d001.s039.t019","gold_sense_keys":["office%1:06:00::"],"candidates":[{"index":1,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":2,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":3,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":4,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":5,"sense_key":"office%1:04:02::","synset_id":"position.n.06"},{"index":6,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":7,"sense_key":"office%1:04:01::","synset_id":"function.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"office%1:06:00::","call_ids":["senseval3.d001.s039.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"office%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":742,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":118},"cost":{"currency":"USD","total_usd":0.007730000000000001,"input_uncached_usd":0.00371,"input_cached_usd":0.0,"output_usd":0.00402,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.54337237495929}
{"item_id":"senseval3.d001.s040.t000","gold_sense_keys":["examination%1:04:00::"],"candidates":[{"index":1,"sense_key":"examination%1:10:00::","synset_id":"examination.n.02"},{"index":2,"sense_key":"examination%1:04:00::","synset_id":"examination.n.01"},{"index":3,"sense_key":"examination%1:10:01::","synset_id":"interrogation.n.03"},{"index":4,"sense_key":"examination%1:04:02::","synset_id":"examination.n.05"},{"index":5,"sense_key":"examination%1:09:03::","synset_id":"examen.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"examination%1:04:00::","call_ids":["senseval3.d001.s040.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"examination%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.0052250000000000005,"input_uncached_usd":0.003305,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6960532079683617}
{"item_id":"senseval3.d001.s040.t001","gold_sense_keys":["change%1:04:00::","change%1:11:00::","change%1:19:00::","change%1:24:00::"],"candidates":[{"index":1,"sense_key":"change%1:04:00::","synset_id":"change.n.03"},{"index":2,"sense_key":"change%1:19:00::","synset_id":"change.n.04"},{"index":3,"sense_key":"change%1:21:03::","synset_id":"change.n.09"},{"index":4,"sense_key":"change%1:07:00::","synset_id":"variety.n.06"},{"index":5,"sense_key":"change%1:21:01::","synset_id":"change.n.08"},{"index":6,"sense_key":"change%1:06:00::","synset_id":"change.n.07"},{"index":7,"sense_key":"change%1:11:00::","synset_id":"change.n.01"},{"index":8,"sense_key":"change%1:06:01::","synset_id":"change.n.06"},{"index":9,"sense_key":"change%1:21:02::","synset_id":"change.n.05"},{"index":10,"sense_key":"change%1:24:00::","synset_id":"change.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"change%1:24:00::","call_ids":["senseval3.d001.s040.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"change%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":892,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.03584,"input_uncached_usd":0.00446,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":15.32327625004109}
{"item_id":"senseval3.d001.s040.t002","gold_sense_keys":["personal%3:00:00::","personal%5:00:00:subjective:00"],"candidates":[{"index":1,"sense_key":"personal%3:01:01::","synset_id":"personal.a.03"},{"index":2,"sense_key":"personal%3:01:00::","synset_id":"personal.a.05"},{"index":3,"sense_key":"personal%5:00:00:subjective:00","synset_id":"personal.s.01"},{"index":4,"sense_key":"personal%5:00:00:physical:00","synset_id":"personal.s.02"},{"index":5,"sense_key":"personal%3:00:00::","synset_id":"personal.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"personal%5:00:00:subjective:00","call_ids":["senseval3.d001.s040.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"personal%5:00:00:subjective:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01895,"input_uncached_usd":0.0031100000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.966296750004403}
{"item_id":"senseval3.d001.s040.t003","gold_sense_keys":["decision%1:09:00::"],"candidates":[{"index":1,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"},{"index":2,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"},{"index":3,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"},{"index":4,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"},{"index":5,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"decision%1:09:00::","call_ids":["senseval3.d001.s040.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"decision%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":445,"reasoning_output_tokens":429},"cost":{"currency":"USD","total_usd":0.016774999999999998,"input_uncached_usd":0.003425,"input_cached_usd":0.0,"output_usd":0.01335,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.543559000012465}
{"item_id":"senseval3.d001.s040.t004","gold_sense_keys":["control%1:04:00::","control%1:07:00::","control%1:24:00::"],"candidates":[{"index":1,"sense_key":"control%1:10:00::","synset_id":"control.n.11"},{"index":2,"sense_key":"control%1:24:00::","synset_id":"control.n.02"},{"index":3,"sense_key":"control%1:04:02::","synset_id":"control.n.03"},{"index":4,"sense_key":"control%1:06:00::","synset_id":"control.n.09"},{"index":5,"sense_key":"control%1:09:01::","synset_id":"command.n.06"},{"index":6,"sense_key":"control%1:04:00::","synset_id":"control.n.05"},{"index":7,"sense_key":"control%1:09:02::","synset_id":"control_condition.n.01"},{"index":8,"sense_key":"control%1:07:00::","synset_id":"control.n.01"},{"index":9,"sense_key":"control%1:07:01::","synset_id":"restraint.n.02"},{"index":10,"sense_key":"control%1:26:00::","synset_id":"dominance.n.02"},{"index":11,"sense_key":"control%1:18:00::","synset_id":"control.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"control%1:07:00::","call_ids":["senseval3.d001.s040.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"control%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":965,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.036205,"input_uncached_usd":0.004825,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":30.456596125033684}
{"item_id":"senseval3.d001.s040.t005","gold_sense_keys":["body%1:14:00::"],"candidates":[{"index":1,"sense_key":"body%1:08:00::","synset_id":"body.n.01"},{"index":2,"sense_key":"body%1:14:00::","synset_id":"body.n.02"},{"index":3,"sense_key":"body%1:17:00::","synset_id":"body.n.04"},{"index":4,"sense_key":"body%1:06:01::","synset_id":"soundbox.n.01"},{"index":5,"sense_key":"body%1:14:02::","synset_id":"body.n.06"},{"index":6,"sense_key":"body%1:06:00::","synset_id":"body.n.11"},{"index":7,"sense_key":"body%1:07:01::","synset_id":"body.n.09"},{"index":8,"sense_key":"body%1:08:01::","synset_id":"torso.n.01"},{"index":9,"sense_key":"body%1:10:00::","synset_id":"body.n.08"},{"index":10,"sense_key":"body%1:08:02::","synset_id":"body.n.03"},{"index":11,"sense_key":"body%1:07:00::","synset_id":"consistency.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"body%1:14:00::","call_ids":["senseval3.d001.s040.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"body%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":914,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.00664,"input_uncached_usd":0.00457,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.956149874953553}
{"item_id":"senseval3.d001.s040.t007","gold_sense_keys":["party%1:14:01::"],"candidates":[{"index":1,"sense_key":"party%1:14:02::","synset_id":"party.n.03"},{"index":2,"sense_key":"party%1:14:00::","synset_id":"party.n.02"},{"index":3,"sense_key":"party%1:11:00::","synset_id":"party.n.04"},{"index":4,"sense_key":"party%1:14:01::","synset_id":"party.n.01"},{"index":5,"sense_key":"party%1:18:00::","synset_id":"party.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"party%1:14:01::","call_ids":["senseval3.d001.s040.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"party%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.00561,"input_uncached_usd":0.00333,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1603811249369755}
{"item_id":"senseval3.d001.s040.t008","gold_sense_keys":["reveal%2:39:00::"],"candidates":[{"index":1,"sense_key":"reveal%2:39:04::","synset_id":"reveal.v.03"},{"index":2,"sense_key":"reveal%2:39:00::","synset_id":"uncover.v.01"},{"index":3,"sense_key":"reveal%2:32:00::","synset_id":"unwrap.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"reveal%2:32:00::","call_ids":["senseval3.d001.s040.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"reveal%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":178,"reasoning_output_tokens":162},"cost":{"currency":"USD","total_usd":0.008485,"input_uncached_usd":0.0031450000000000002,"input_cached_usd":0.0,"output_usd":0.00534,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.214765083044767}
{"item_id":"senseval3.d001.s040.t009","gold_sense_keys":["personal%3:00:00::","personal%5:00:00:subjective:00"],"candidates":[{"index":1,"sense_key":"personal%3:01:01::","synset_id":"personal.a.03"},{"index":2,"sense_key":"personal%3:01:00::","synset_id":"personal.a.05"},{"index":3,"sense_key":"personal%3:00:00::","synset_id":"personal.a.01"},{"index":4,"sense_key":"personal%5:00:00:subjective:00","synset_id":"personal.s.01"},{"index":5,"sense_key":"personal%5:00:00:physical:00","synset_id":"personal.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"personal%3:00:00::","call_ids":["senseval3.d001.s040.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"personal%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":818,"reasoning_output_tokens":796},"cost":{"currency":"USD","total_usd":0.027650000000000004,"input_uncached_usd":0.0031100000000000004,"input_cached_usd":0.0,"output_usd":0.02454,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.152757584000938}
{"item_id":"senseval3.d001.s040.t010","gold_sense_keys":["difference%1:07:00::"],"candidates":[{"index":1,"sense_key":"difference%1:24:00::","synset_id":"difference.n.04"},{"index":2,"sense_key":"difference%1:11:00::","synset_id":"deviation.n.01"},{"index":3,"sense_key":"difference%1:10:00::","synset_id":"dispute.n.01"},{"index":4,"sense_key":"difference%1:23:00::","synset_id":"remainder.n.03"},{"index":5,"sense_key":"difference%1:07:00::","synset_id":"difference.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"difference%1:07:00::","call_ids":["senseval3.d001.s040.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"difference%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":675,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":101},"cost":{"currency":"USD","total_usd":0.0068850000000000005,"input_uncached_usd":0.0033750000000000004,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9665987089974806}
{"item_id":"senseval3.d001.s040.t011","gold_sense_keys":["note%2:32:00::"],"candidates":[{"index":1,"sense_key":"note%2:32:01::","synset_id":"note.v.04"},{"index":2,"sense_key":"note%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"note%2:39:02::","synset_id":"note.v.03"},{"index":4,"sense_key":"note%2:32:00::","synset_id":"note.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"note%2:32:00::","call_ids":["senseval3.d001.s040.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"note%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":241,"reasoning_output_tokens":225},"cost":{"currency":"USD","total_usd":0.01042,"input_uncached_usd":0.00319,"input_cached_usd":0.0,"output_usd":0.00723,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.8794550829334185}
{"item_id":"senseval3.d001.s040.t012","gold_sense_keys":["table%1:14:00::"],"candidates":[{"index":1,"sense_key":"table%1:13:01::","synset_id":"board.n.04"},{"index":2,"sense_key":"table%1:14:00::","synset_id":"table.n.01"},{"index":3,"sense_key":"table%1:14:01::","synset_id":"table.n.05"},{"index":4,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":5,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":6,"sense_key":"table%1:06:02::","synset_id":"table.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"table%1:14:00::","call_ids":["senseval3.d001.s040.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"table%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":714,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.0053100000000000005,"input_uncached_usd":0.0035700000000000003,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4910695840371773}
{"item_id":"senseval3.d001.s040.t013","gold_sense_keys":["attribute%2:31:00::"],"candidates":[{"index":1,"sense_key":"attribute%2:31:01::","synset_id":"assign.v.08"},{"index":2,"sense_key":"attribute%2:31:00::","synset_id":"impute.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"attribute%2:31:00::","call_ids":["senseval3.d001.s040.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"attribute%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.0049250000000000006,"input_uncached_usd":0.002795,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6171684169676155}
{"item_id":"senseval3.d001.s040.t015","gold_sense_keys":["control%1:07:00::"],"candidates":[{"index":1,"sense_key":"control%1:06:00::","synset_id":"control.n.09"},{"index":2,"sense_key":"control%1:09:02::","synset_id":"control_condition.n.01"},{"index":3,"sense_key":"control%1:04:00::","synset_id":"control.n.05"},{"index":4,"sense_key":"control%1:10:00::","synset_id":"control.n.11"},{"index":5,"sense_key":"control%1:26:00::","synset_id":"dominance.n.02"},{"index":6,"sense_key":"control%1:24:00::","synset_id":"control.n.02"},{"index":7,"sense_key":"control%1:07:00::","synset_id":"control.n.01"},{"index":8,"sense_key":"control%1:07:01::","synset_id":"restraint.n.02"},{"index":9,"sense_key":"control%1:04:02::","synset_id":"control.n.03"},{"index":10,"sense_key":"control%1:09:01::","synset_id":"command.n.06"},{"index":11,"sense_key":"control%1:18:00::","synset_id":"control.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"control%1:07:00::","call_ids":["senseval3.d001.s040.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"control%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":965,"cached_input_tokens":0,"output_tokens":520,"reasoning_output_tokens":504},"cost":{"currency":"USD","total_usd":0.020425,"input_uncached_usd":0.004825,"input_cached_usd":0.0,"output_usd":0.015600000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.682030832977034}
{"item_id":"senseval3.d001.s040.t016","gold_sense_keys":["democrat%1:18:01::"],"candidates":[{"index":1,"sense_key":"democrat%1:18:00::","synset_id":"democrat.n.02"},{"index":2,"sense_key":"democrat%1:18:01::","synset_id":"democrat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democrat%1:18:01::","call_ids":["senseval3.d001.s040.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democrat%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.0046,"input_uncached_usd":0.0025900000000000003,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2202103339368477}
{"item_id":"senseval3.d001.s040.t017","gold_sense_keys":["exercise%2:34:00::"],"candidates":[{"index":1,"sense_key":"exercise%2:29:00::","synset_id":"exercise.v.04"},{"index":2,"sense_key":"exercise%2:34:00::","synset_id":"exert.v.01"},{"index":3,"sense_key":"exercise%2:29:01::","synset_id":"exercise.v.03"},{"index":4,"sense_key":"exercise%2:41:00::","synset_id":"practice.v.01"},{"index":5,"sense_key":"exercise%2:31:00::","synset_id":"drill.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"exercise%2:34:00::","call_ids":["senseval3.d001.s040.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"exercise%2:34:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":660,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.00561,"input_uncached_usd":0.0033000000000000004,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9190937090897933}
{"item_id":"senseval3.d001.s040.t019","gold_sense_keys":["status%1:26:00::"],"candidates":[{"index":1,"sense_key":"status%1:26:01::","synset_id":"condition.n.01"},{"index":2,"sense_key":"status%1:26:00::","synset_id":"status.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"status%1:26:00::","call_ids":["senseval3.d001.s040.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"status%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":85},"cost":{"currency":"USD","total_usd":0.0058000000000000005,"input_uncached_usd":0.0028000000000000004,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.367592207971029}
{"item_id":"senseval3.d001.s040.t020","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":2,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":3,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":4,"sense_key":"other%3:00:00::","synset_id":"other.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d001.s040.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0049050000000000005,"input_uncached_usd":0.0031950000000000004,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6805006659124047}
{"item_id":"senseval3.d001.s040.t021","gold_sense_keys":["resource%1:07:00::"],"candidates":[{"index":1,"sense_key":"resource%1:21:00::","synset_id":"resource.n.01"},{"index":2,"sense_key":"resource%1:09:00::","synset_id":"resource.n.03"},{"index":3,"sense_key":"resource%1:07:00::","synset_id":"resource.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"resource%1:07:00::","call_ids":["senseval3.d001.s040.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"resource%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.006125,"input_uncached_usd":0.0028850000000000004,"input_cached_usd":0.0,"output_usd":0.0032400000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1321162909734994}
{"item_id":"senseval3.d001.s040.t022","gold_sense_keys":["committee%1:14:00::"],"candidates":[{"index":1,"sense_key":"committee%1:14:01::","synset_id":"committee.n.02"},{"index":2,"sense_key":"committee%1:14:00::","synset_id":"committee.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"committee%1:14:00::","call_ids":["senseval3.d001.s040.t022__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"committee%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004515,"input_uncached_usd":0.0027150000000000004,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.213036041939631}
{"item_id":"senseval3.d001.s040.t023","gold_sense_keys":["staff%1:14:01::"],"candidates":[{"index":1,"sense_key":"staff%1:10:01::","synset_id":"staff.n.05"},{"index":2,"sense_key":"staff%1:14:01::","synset_id":"staff.n.01"},{"index":3,"sense_key":"staff%1:10:00::","synset_id":"staff.n.06"},{"index":4,"sense_key":"staff%1:27:00::","synset_id":"staff.n.04"},{"index":5,"sense_key":"staff%1:06:00::","synset_id":"staff.n.02"},{"index":6,"sense_key":"staff%1:14:00::","synset_id":"staff.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"staff%1:14:01::","call_ids":["senseval3.d001.s040.t023__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"staff%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.005955,"input_uncached_usd":0.0034950000000000003,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1576650409260765}
{"item_id":"senseval3.d001.s041.t001","gold_sense_keys":["piece%1:06:00::"],"candidates":[{"index":1,"sense_key":"piece%1:17:00::","synset_id":"part.n.03"},{"index":2,"sense_key":"piece%1:07:00::","synset_id":"piece.n.09"},{"index":3,"sense_key":"piece%1:06:02::","synset_id":"man.n.10"},{"index":4,"sense_key":"piece%1:11:00::","synset_id":"piece.n.05"},{"index":5,"sense_key":"piece%1:06:00::","synset_id":"piece.n.01"},{"index":6,"sense_key":"piece%1:10:02::","synset_id":"piece.n.06"},{"index":7,"sense_key":"piece%1:21:00::","synset_id":"slice.n.01"},{"index":8,"sense_key":"piece%1:06:05::","synset_id":"piece.n.02"},{"index":9,"sense_key":"piece%1:13:00::","synset_id":"piece.n.08"},{"index":10,"sense_key":"piece%1:10:01::","synset_id":"musical_composition.n.01"},{"index":11,"sense_key":"piece%1:28:00::","synset_id":"while.n.01"},{"index":12,"sense_key":"piece%1:06:03::","synset_id":"firearm.n.01"},{"index":13,"sense_key":"piece%1:06:04::","synset_id":"objet_d'art.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"piece%1:06:00::","call_ids":["senseval3.d001.s041.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"piece%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1021,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":85},"cost":{"currency":"USD","total_usd":0.008105000000000001,"input_uncached_usd":0.005105,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.555111291934736}
{"item_id":"senseval3.d001.s041.t002","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"},{"index":2,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":3,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval3.d001.s041.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":399,"reasoning_output_tokens":383},"cost":{"currency":"USD","total_usd":0.014884999999999999,"input_uncached_usd":0.002915,"input_cached_usd":0.0,"output_usd":0.01197,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.488955792039633}
{"item_id":"senseval3.d001.s041.t003","gold_sense_keys":["hold%2:42:00::"],"candidates":[{"index":1,"sense_key":"hold%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"hold%2:42:08::","synset_id":"defy.v.01"},{"index":3,"sense_key":"hold%2:40:04::","synset_id":"bear.v.11"},{"index":4,"sense_key":"hold%2:35:09::","synset_id":"hold.v.31"},{"index":5,"sense_key":"hold%2:37:00::","synset_id":"harbor.v.01"},{"index":6,"sense_key":"hold%2:42:01::","synset_id":"hold.v.13"},{"index":7,"sense_key":"hold%2:32:13::","synset_id":"oblige.v.02"},{"index":8,"sense_key":"hold%2:32:15::","synset_id":"agree.v.01"},{"index":9,"sense_key":"hold%2:42:05::","synset_id":"accommodate.v.04"},{"index":10,"sense_key":"hold%2:40:02::","synset_id":"retain.v.03"},{"index":11,"sense_key":"hold%2:35:01::","synset_id":"hold.v.10"},{"index":12,"sense_key":"hold%2:31:01::","synset_id":"deem.v.01"},{"index":13,"sense_key":"hold%2:31:08::","synset_id":"hold.v.16"},{"index":14,"sense_key":"hold%2:42:02::","synset_id":"prevail.v.02"},{"index":15,"sense_key":"hold%2:33:00::","synset_id":"hold.v.33"},{"index":16,"sense_key":"hold%2:41:15::","synset_id":"control.v.02"},{"index":17,"sense_key":"hold%2:38:05::","synset_id":"halt.v.01"},{"index":18,"sense_key":"hold%2:42:07::","synset_id":"hold.v.17"},{"index":19,"sense_key":"hold%2:42:12::","synset_id":"hold.v.26"},{"index":20,"sense_key":"hold%2:35:02::","synset_id":"hold.v.14"},{"index":21,"sense_key":"hold%2:34:00::","synset_id":"carry.v.33"},{"index":22,"sense_key":"hold%2:31:04::","synset_id":"hold.v.22"},{"index":23,"sense_key":"hold%2:35:03::","synset_id":"restrain.v.03"},{"index":24,"sense_key":"hold%2:32:11::","synset_id":"declare.v.04"},{"index":25,"sense_key":"hold%2:42:00::","synset_id":"keep.v.01"},{"index":26,"sense_key":"hold%2:36:00::","synset_id":"hold.v.03"},{"index":27,"sense_key":"hold%2:41:14::","synset_id":"hold.v.29"},{"index":28,"sense_key":"hold%2:42:14::","synset_id":"contain.v.05"},{"index":29,"sense_key":"hold%2:33:05::","synset_id":"defend.v.03"},{"index":30,"sense_key":"hold%2:29:04::","synset_id":"hold.v.36"},{"index":31,"sense_key":"hold%2:35:00::","synset_id":"hold.v.02"},{"index":32,"sense_key":"hold%2:42:04::","synset_id":"apply.v.02"},{"index":33,"sense_key":"hold%2:41:13::","synset_id":"hold.v.28"},{"index":34,"sense_key":"hold%2:41:00::","synset_id":"reserve.v.04"},{"index":35,"sense_key":"hold%2:42:13::","synset_id":"hold.v.11"},{"index":36,"sense_key":"hold%2:31:10::","synset_id":"hold.v.23"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":25,"chosen_sense_key":"hold%2:42:00::","call_ids":["senseval3.d001.s041.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":25,"predicted_sense_key":"hold%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2088,"cached_input_tokens":0,"output_tokens":179,"reasoning_output_tokens":163},"cost":{"currency":"USD","total_usd":0.01581,"input_uncached_usd":0.010440000000000001,"input_cached_usd":0.0,"output_usd":0.00537,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.946743540931493}
{"item_id":"senseval3.d001.s041.t004","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":2,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":3,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":4,"sense_key":"other%3:00:00::","synset_id":"other.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d001.s041.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":635,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.005485,"input_uncached_usd":0.0031750000000000003,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8713717500213534}
{"item_id":"senseval3.d001.s041.t005","gold_sense_keys":["factor%1:11:00::"],"candidates":[{"index":1,"sense_key":"factor%1:09:00::","synset_id":"component.n.01"},{"index":2,"sense_key":"factor%1:11:00::","synset_id":"factor.n.01"},{"index":3,"sense_key":"factor%1:09:01::","synset_id":"factor.n.06"},{"index":4,"sense_key":"factor%1:08:00::","synset_id":"gene.n.01"},{"index":5,"sense_key":"factor%1:23:01::","synset_id":"factor.n.05"},{"index":6,"sense_key":"factor%1:18:00::","synset_id":"agent.n.04"},{"index":7,"sense_key":"factor%1:23:00::","synset_id":"divisor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"factor%1:09:01::","call_ids":["senseval3.d001.s041.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"factor%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":315,"reasoning_output_tokens":299},"cost":{"currency":"USD","total_usd":0.013285,"input_uncached_usd":0.0038350000000000003,"input_cached_usd":0.0,"output_usd":0.00945,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.602545417030342}
{"item_id":"senseval3.d001.s041.t006","gold_sense_keys":["constant%5:00:00:invariable:00"],"candidates":[{"index":1,"sense_key":"constant%5:00:00:invariable:00","synset_id":"changeless.s.01"},{"index":2,"sense_key":"constant%3:00:00::","synset_id":"constant.a.02"},{"index":3,"sense_key":"constant%5:00:00:continuous:01","synset_id":"ceaseless.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"constant%5:00:00:invariable:00","call_ids":["senseval3.d001.s041.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"constant%5:00:00:invariable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.0047,"input_uncached_usd":0.0030800000000000003,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.480831041932106}
{"item_id":"senseval3.d001.s041.t007","gold_sense_keys":["incumbency%1:04:00::","incumbency%1:28:00::"],"candidates":[{"index":1,"sense_key":"incumbency%1:04:00::","synset_id":"incumbency.n.03"},{"index":2,"sense_key":"incumbency%1:28:00::","synset_id":"tenure.n.01"},{"index":3,"sense_key":"incumbency%1:04:01::","synset_id":"incumbency.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"incumbency%1:04:00::","call_ids":["senseval3.d001.s041.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"incumbency%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":147},"cost":{"currency":"USD","total_usd":0.007635,"input_uncached_usd":0.002745,"input_cached_usd":0.0,"output_usd":0.00489,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0085297089535743}
{"item_id":"senseval3.d001.s041.t008","gold_sense_keys":["advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"},{"index":2,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"},{"index":3,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"advantage%1:07:00::","call_ids":["senseval3.d001.s041.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"advantage%1:07:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018725,"input_uncached_usd":0.0028850000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.340461250045337}
{"item_id":"senseval3.d001.s041.t009","gold_sense_keys":["regional%5:00:00:territorial:00"],"candidates":[{"index":1,"sense_key":"regional%3:01:00::","synset_id":"regional.a.01"},{"index":2,"sense_key":"regional%5:00:00:territorial:00","synset_id":"regional.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"regional%3:01:00::","call_ids":["senseval3.d001.s041.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"regional%3:01:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018415,"input_uncached_usd":0.002605,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.519910083035938}
{"item_id":"senseval3.d001.s041.t010","gold_sense_keys":["factor%1:11:00::"],"candidates":[{"index":1,"sense_key":"factor%1:09:00::","synset_id":"component.n.01"},{"index":2,"sense_key":"factor%1:18:00::","synset_id":"agent.n.04"},{"index":3,"sense_key":"factor%1:08:00::","synset_id":"gene.n.01"},{"index":4,"sense_key":"factor%1:11:00::","synset_id":"factor.n.01"},{"index":5,"sense_key":"factor%1:23:00::","synset_id":"divisor.n.01"},{"index":6,"sense_key":"factor%1:09:01::","synset_id":"factor.n.06"},{"index":7,"sense_key":"factor%1:23:01::","synset_id":"factor.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"factor%1:09:01::","call_ids":["senseval3.d001.s041.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"factor%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":2082,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.06629,"input_uncached_usd":0.0038300000000000005,"input_cached_usd":0.0,"output_usd":0.06246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":31.248895333032124}
{"item_id":"senseval3.d001.s041.t011","gold_sense_keys":["difference%1:23:00::"],"candidates":[{"index":1,"sense_key":"difference%1:07:00::","synset_id":"difference.n.01"},{"index":2,"sense_key":"difference%1:24:00::","synset_id":"difference.n.04"},{"index":3,"sense_key":"difference%1:23:00::","synset_id":"remainder.n.03"},{"index":4,"sense_key":"difference%1:10:00::","synset_id":"dispute.n.01"},{"index":5,"sense_key":"difference%1:11:00::","synset_id":"deviation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"difference%1:23:00::","call_ids":["senseval3.d001.s041.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"difference%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":670,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.0062900000000000005,"input_uncached_usd":0.00335,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3158200409961864}
{"item_id":"senseval3.d001.s041.t012","gold_sense_keys":["popular%5:00:00:democratic:00"],"candidates":[{"index":1,"sense_key":"popular%5:00:00:nonclassical:00","synset_id":"popular.s.03"},{"index":2,"sense_key":"popular%3:00:00::","synset_id":"popular.a.01"},{"index":3,"sense_key":"popular%5:00:01:common:01","synset_id":"democratic.s.01"},{"index":4,"sense_key":"popular%5:00:00:democratic:00","synset_id":"popular.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"popular%5:00:00:democratic:00","call_ids":["senseval3.d001.s041.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"popular%5:00:00:democratic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":642,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.00555,"input_uncached_usd":0.00321,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3794752919347957}
{"item_id":"senseval3.d001.s041.t013","gold_sense_keys":["vote%1:04:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":2,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"},{"index":3,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":4,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":5,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"vote%1:04:00::","call_ids":["senseval3.d001.s041.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"vote%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":243,"reasoning_output_tokens":227},"cost":{"currency":"USD","total_usd":0.010765,"input_uncached_usd":0.0034750000000000002,"input_cached_usd":0.0,"output_usd":0.0072900000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9876139590051025}
{"item_id":"senseval3.d001.s041.t014","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s041.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.0047550000000000005,"input_uncached_usd":0.002925,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8861300830030814}
{"item_id":"senseval3.d001.s041.t015","gold_sense_keys":["presidential%3:01:00::"],"candidates":[{"index":1,"sense_key":"presidential%3:01:00::","synset_id":"presidential.a.01"},{"index":2,"sense_key":"presidential%3:00:00::","synset_id":"presidential.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"presidential%3:01:00::","call_ids":["senseval3.d001.s041.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"presidential%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.004685,"input_uncached_usd":0.002645,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1426145000150427}
{"item_id":"senseval3.d001.s041.t017","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s041.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.004995,"input_uncached_usd":0.0026550000000000002,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.652251915889792}
{"item_id":"senseval3.d001.s041.t018","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":2,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":6,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":7,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":8,"sense_key":"state%1:15:01::","synset_id":"state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"state%1:15:01::","call_ids":["senseval3.d001.s041.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"state%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.0065450000000000005,"input_uncached_usd":0.0042650000000000006,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6831442089751363}
{"item_id":"senseval3.d001.s041.t019","gold_sense_keys":["conduct%2:41:00::"],"candidates":[{"index":1,"sense_key":"conduct%2:36:00::","synset_id":"conduct.v.02"},{"index":2,"sense_key":"conduct%2:38:01::","synset_id":"lead.v.01"},{"index":3,"sense_key":"conduct%2:41:00::","synset_id":"conduct.v.01"},{"index":4,"sense_key":"conduct%2:41:01::","synset_id":"behave.v.02"},{"index":5,"sense_key":"conduct%2:36:04::","synset_id":"conduct.v.06"},{"index":6,"sense_key":"conduct%2:38:00::","synset_id":"impart.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"conduct%2:41:00::","call_ids":["senseval3.d001.s041.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"conduct%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":748,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.0062900000000000005,"input_uncached_usd":0.0037400000000000003,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.381448291009292}
{"item_id":"senseval3.d001.s041.t020","gold_sense_keys":["election%1:04:01::"],"candidates":[{"index":1,"sense_key":"election%1:04:00::","synset_id":"election.n.02"},{"index":2,"sense_key":"election%1:09:00::","synset_id":"election.n.04"},{"index":3,"sense_key":"election%1:04:01::","synset_id":"election.n.01"},{"index":4,"sense_key":"election%1:26:00::","synset_id":"election.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"election%1:04:01::","call_ids":["senseval3.d001.s041.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"election%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":608,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.00442,"input_uncached_usd":0.00304,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.789020458003506}
{"item_id":"senseval3.d001.s041.t021","gold_sense_keys":["turn_out%2:42:00::","turn_out%2:42:01::"],"candidates":[{"index":1,"sense_key":"turn_out%2:38:00::","synset_id":"turn_out.v.10"},{"index":2,"sense_key":"turn_out%2:36:01::","synset_id":"turn_out.v.03"},{"index":3,"sense_key":"turn_out%2:35:01::","synset_id":"switch_off.v.01"},{"index":4,"sense_key":"turn_out%2:41:03::","synset_id":"turn_out.v.08"},{"index":5,"sense_key":"turn_out%2:36:00::","synset_id":"bear.v.05"},{"index":6,"sense_key":"turn_out%2:42:01::","synset_id":"turn_out.v.02"},{"index":7,"sense_key":"turn_out%2:42:00::","synset_id":"prove.v.01"},{"index":8,"sense_key":"turn_out%2:29:00::","synset_id":"get_up.v.02"},{"index":9,"sense_key":"turn_out%2:35:00::","synset_id":"eject.v.01"},{"index":10,"sense_key":"turn_out%2:40:03::","synset_id":"turn_out.v.09"},{"index":11,"sense_key":"turn_out%2:30:00::","synset_id":"turn_out.v.05"},{"index":12,"sense_key":"turn_out%2:42:02::","synset_id":"turn_out.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"turn_out%2:42:00::","call_ids":["senseval3.d001.s041.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"turn_out%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":984,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02076,"input_uncached_usd":0.004920000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.284830207936466}
{"item_id":"senseval3.d001.s041.t022","gold_sense_keys":["positive%5:00:00:plus:00"],"candidates":[{"index":1,"sense_key":"positive%3:01:00::","synset_id":"positivist.a.01"},{"index":2,"sense_key":"positive%5:00:00:confident:00","synset_id":"cocksure.s.01"},{"index":3,"sense_key":"positive%5:00:00:advantageous:00","synset_id":"plus.s.01"},{"index":4,"sense_key":"positive%3:00:01::","synset_id":"positive.a.01"},{"index":5,"sense_key":"positive%5:00:00:certain:02","synset_id":"convinced.s.01"},{"index":6,"sense_key":"positive%5:00:00:formal:01","synset_id":"positive.s.03"},{"index":7,"sense_key":"positive%3:00:05::","synset_id":"positive.a.08"},{"index":8,"sense_key":"positive%5:00:00:undeniable:00","synset_id":"incontrovertible.s.01"},{"index":9,"sense_key":"positive%5:00:00:plus:00","synset_id":"positive.s.05"},{"index":10,"sense_key":"positive%3:00:04::","synset_id":"positive.a.04"},{"index":11,"sense_key":"positive%5:00:00:charged:00","synset_id":"positive.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"positive%3:00:05::","call_ids":["senseval3.d001.s041.t022__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"positive%3:00:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":943,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020555,"input_uncached_usd":0.0047150000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.352194875013083}
{"item_id":"senseval3.d001.s041.t023","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":2,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":3,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":4,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":5,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":6,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"},{"index":7,"sense_key":"function%1:24:01::","synset_id":"function.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"function%1:24:01::","call_ids":["senseval3.d001.s041.t023__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"function%1:24:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":799,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.007235,"input_uncached_usd":0.003995,"input_cached_usd":0.0,"output_usd":0.0032400000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6861062080133706}
{"item_id":"senseval3.d001.s041.t027","gold_sense_keys":["burden%1:09:01::"],"candidates":[{"index":1,"sense_key":"burden%1:10:00::","synset_id":"effect.n.04"},{"index":2,"sense_key":"burden%1:09:01::","synset_id":"burden.n.01"},{"index":3,"sense_key":"burden%1:06:01::","synset_id":"load.n.01"},{"index":4,"sense_key":"burden%1:09:00::","synset_id":"burden.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"burden%1:09:01::","call_ids":["senseval3.d001.s041.t027__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"burden%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":268,"reasoning_output_tokens":252},"cost":{"currency":"USD","total_usd":0.011105,"input_uncached_usd":0.0030650000000000004,"input_cached_usd":0.0,"output_usd":0.00804,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.6798439999111}
{"item_id":"senseval3.d001.s041.t028","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":3,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":4,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":5,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"},{"index":6,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":7,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":8,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"state%1:15:01::","call_ids":["senseval3.d001.s041.t028__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"state%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.006995,"input_uncached_usd":0.0042650000000000006,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.929192499956116}
{"item_id":"senseval3.d001.s041.t029","gold_sense_keys":["progressive%3:00:02::"],"candidates":[{"index":1,"sense_key":"progressive%5:00:00:active:02","synset_id":"progressive.s.04"},{"index":2,"sense_key":"progressive%5:00:00:liberal:00","synset_id":"progressive.s.01"},{"index":3,"sense_key":"progressive%5:00:00:increasing:00","synset_id":"progressive.s.02"},{"index":4,"sense_key":"progressive%3:00:01::","synset_id":"progressive.a.01"},{"index":5,"sense_key":"progressive%5:00:00:ordered:00","synset_id":"progressive.s.03"},{"index":6,"sense_key":"progressive%3:00:02::","synset_id":"progressive.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"progressive%3:00:02::","call_ids":["senseval3.d001.s041.t029__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"progressive%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":678,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00519,"input_uncached_usd":0.0033900000000000002,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.692680499982089}
{"item_id":"senseval3.d001.s041.t031","gold_sense_keys":["hit%2:35:05::"],"candidates":[{"index":1,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":2,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"},{"index":3,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":4,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":5,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"},{"index":6,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":7,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":8,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"},{"index":9,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"},{"index":10,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":11,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":12,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":13,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":14,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"},{"index":15,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":16,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"},{"index":17,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"hit%2:35:05::","call_ids":["senseval3.d001.s041.t031__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"hit%2:35:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1199,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.008755,"input_uncached_usd":0.005995,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1836866249796003}
{"item_id":"senseval3.d001.s041.t032","gold_sense_keys":["high%3:00:02::"],"candidates":[{"index":1,"sense_key":"high%5:00:00:malodorous:00","synset_id":"gamey.s.02"},{"index":2,"sense_key":"high%5:00:00:superior:01","synset_id":"eminent.s.01"},{"index":3,"sense_key":"high%5:00:00:elated:00","synset_id":"high.s.02"},{"index":4,"sense_key":"high%3:00:02::","synset_id":"high.a.01"},{"index":5,"sense_key":"high%5:00:00:intoxicated:00","synset_id":"high.s.04"},{"index":6,"sense_key":"high%3:00:01::","synset_id":"high.a.02"},{"index":7,"sense_key":"high%3:00:03::","synset_id":"high.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"high%3:00:02::","call_ids":["senseval3.d001.s041.t032__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"high%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":739,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.006845,"input_uncached_usd":0.0036950000000000004,"input_cached_usd":0.0,"output_usd":0.00315,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.713404041947797}
{"item_id":"senseval3.d001.s041.t034","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"},{"index":2,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":3,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":6,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":7,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":8,"sense_key":"state%1:15:01::","synset_id":"state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"state%1:15:01::","call_ids":["senseval3.d001.s041.t034__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"state%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":114},"cost":{"currency":"USD","total_usd":0.008165,"input_uncached_usd":0.0042650000000000006,"input_cached_usd":0.0,"output_usd":0.0039000000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.54031216702424}
{"item_id":"senseval3.d001.s042.t000","gold_sense_keys":["put%2:32:00::"],"candidates":[{"index":1,"sense_key":"put%2:31:00::","synset_id":"arrange.v.07"},{"index":2,"sense_key":"put%2:34:00::","synset_id":"put.v.04"},{"index":3,"sense_key":"put%2:35:00::","synset_id":"put.v.01"},{"index":4,"sense_key":"put%2:35:05::","synset_id":"put.v.02"},{"index":5,"sense_key":"put%2:31:13::","synset_id":"place.v.11"},{"index":6,"sense_key":"put%2:39:13::","synset_id":"put.v.07"},{"index":7,"sense_key":"put%2:36:13::","synset_id":"put.v.08"},{"index":8,"sense_key":"put%2:40:00::","synset_id":"invest.v.01"},{"index":9,"sense_key":"put%2:32:00::","synset_id":"frame.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"put%2:32:00::","call_ids":["senseval3.d001.s042.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"put%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":804,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.0066300000000000005,"input_uncached_usd":0.00402,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.098808291950263}
{"item_id":"senseval3.d001.s042.t001","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s042.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.00472,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5242852499941364}
{"item_id":"senseval3.d001.s042.t002","gold_sense_keys":["president%1:04:04::"],"candidates":[{"index":1,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":2,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":3,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"president%1:04:04::","call_ids":["senseval3.d001.s042.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"president%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":670,"cached_input_tokens":0,"output_tokens":435,"reasoning_output_tokens":420},"cost":{"currency":"USD","total_usd":0.0164,"input_uncached_usd":0.00335,"input_cached_usd":0.0,"output_usd":0.01305,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.408551583997905}
{"item_id":"senseval3.d001.s042.t003","gold_sense_keys":["look_on%2:31:00::"],"candidates":[{"index":1,"sense_key":"look_on%2:31:00::","synset_id":"think_of.v.03"},{"index":2,"sense_key":"look_on%2:39:00::","synset_id":"watch.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"look_on%2:31:00::","call_ids":["senseval3.d001.s042.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"look_on%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.005415,"input_uncached_usd":0.0026550000000000002,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.172284416970797}
{"item_id":"senseval3.d001.s042.t005","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"},{"index":2,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s042.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":554,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.00481,"input_uncached_usd":0.0027700000000000003,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1120715419529006}
{"item_id":"senseval3.d001.s042.t006","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s042.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.0042699999999999995,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.810676584020257}
{"item_id":"senseval3.d001.s042.t009","gold_sense_keys":["severe%5:00:02:nonindulgent:00"],"candidates":[{"index":1,"sense_key":"severe%5:00:01:bad:00","synset_id":"severe.s.06"},{"index":2,"sense_key":"severe%5:00:00:plain:01","synset_id":"austere.s.01"},{"index":3,"sense_key":"severe%5:00:00:critical:03","synset_id":"dangerous.s.01"},{"index":4,"sense_key":"severe%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":5,"sense_key":"severe%5:00:02:nonindulgent:00","synset_id":"severe.s.04"},{"index":6,"sense_key":"severe%5:00:00:intense:00","synset_id":"severe.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"severe%5:00:02:nonindulgent:00","call_ids":["senseval3.d001.s042.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"severe%5:00:02:nonindulgent:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019445,"input_uncached_usd":0.003605,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.512882207985967}
{"item_id":"senseval3.d001.s043.t000","gold_sense_keys":["appear%2:39:00::"],"candidates":[{"index":1,"sense_key":"appear%2:41:03::","synset_id":"appear.v.07"},{"index":2,"sense_key":"appear%2:30:02::","synset_id":"appear.v.05"},{"index":3,"sense_key":"appear%2:39:01::","synset_id":"appear.v.04"},{"index":4,"sense_key":"appear%2:39:00::","synset_id":"look.v.02"},{"index":5,"sense_key":"appear%2:36:00::","synset_id":"appear.v.06"},{"index":6,"sense_key":"appear%2:30:00::","synset_id":"appear.v.02"},{"index":7,"sense_key":"appear%2:30:01::","synset_id":"appear.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"appear%2:39:00::","call_ids":["senseval3.d001.s043.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"appear%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":781,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019745,"input_uncached_usd":0.0039050000000000005,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.094012457993813}
{"item_id":"senseval3.d001.s043.t001","gold_sense_keys":["take%2:30:01::"],"candidates":[{"index":1,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":2,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":3,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"},{"index":4,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":5,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":6,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":7,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":8,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":9,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":10,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":11,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":12,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":13,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":14,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":15,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":16,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":17,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":18,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":19,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":20,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":21,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":22,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":23,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":24,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":25,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":26,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":27,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":28,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":29,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":30,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":31,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":32,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":35,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":36,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":37,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":38,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":39,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":42,"sense_key":"take%2:40:01::","synset_id":"take.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"take%2:30:01::","call_ids":["senseval3.d001.s043.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"take%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2298,"cached_input_tokens":0,"output_tokens":223,"reasoning_output_tokens":208},"cost":{"currency":"USD","total_usd":0.01818,"input_uncached_usd":0.01149,"input_cached_usd":0.0,"output_usd":0.00669,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7078381249448285}
{"item_id":"senseval3.d001.s043.t002","gold_sense_keys":["same%3:00:00::"],"candidates":[{"index":1,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":2,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"},{"index":3,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":4,"sense_key":"same%3:00:04::","synset_id":"like.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"same%3:00:00::","call_ids":["senseval3.d001.s043.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"same%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018825,"input_uncached_usd":0.0029850000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.893836791045032}
{"item_id":"senseval3.d001.s043.t007","gold_sense_keys":["level%1:26:01::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":3,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":4,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":5,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"level%1:26:01::","call_ids":["senseval3.d001.s043.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"level%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":797,"cached_input_tokens":0,"output_tokens":851,"reasoning_output_tokens":829},"cost":{"currency":"USD","total_usd":0.029515,"input_uncached_usd":0.003985000000000001,"input_cached_usd":0.0,"output_usd":0.02553,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.713365333038382}
{"item_id":"senseval3.d001.s043.t008","gold_sense_keys":["federal%5:00:00:national:01"],"candidates":[{"index":1,"sense_key":"federal%3:01:02::","synset_id":"federal.a.02"},{"index":2,"sense_key":"federal%5:00:00:national:01","synset_id":"federal.s.01"},{"index":3,"sense_key":"federal%3:00:00::","synset_id":"federal.a.04"},{"index":4,"sense_key":"federal%5:00:00:northern:02","synset_id":"union.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"federal%5:00:00:national:01","call_ids":["senseval3.d001.s043.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"federal%5:00:00:national:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":644,"cached_input_tokens":0,"output_tokens":366,"reasoning_output_tokens":351},"cost":{"currency":"USD","total_usd":0.0142,"input_uncached_usd":0.00322,"input_cached_usd":0.0,"output_usd":0.01098,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.921341542038135}
{"item_id":"senseval3.d001.s043.t009","gold_sense_keys":["level%1:26:01::"],"candidates":[{"index":1,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":2,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":3,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":4,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":5,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":6,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":7,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"level%1:26:01::","call_ids":["senseval3.d001.s043.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"level%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":796,"cached_input_tokens":0,"output_tokens":437,"reasoning_output_tokens":422},"cost":{"currency":"USD","total_usd":0.01709,"input_uncached_usd":0.00398,"input_cached_usd":0.0,"output_usd":0.01311,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.706311917048879}
{"item_id":"senseval3.d001.s044.t003","gold_sense_keys":["line%1:09:00::"],"candidates":[{"index":1,"sense_key":"line%1:06:07::","synset_id":"telephone_line.n.02"},{"index":2,"sense_key":"line%1:06:00::","synset_id":"line.n.18"},{"index":3,"sense_key":"line%1:15:01::","synset_id":"line.n.20"},{"index":4,"sense_key":"line%1:23:00::","synset_id":"agate_line.n.01"},{"index":5,"sense_key":"line%1:19:00::","synset_id":"line.n.06"},{"index":6,"sense_key":"line%1:06:08::","synset_id":"line.n.22"},{"index":7,"sense_key":"line%1:06:09::","synset_id":"production_line.n.01"},{"index":8,"sense_key":"line%1:10:03::","synset_id":"tune.n.01"},{"index":9,"sense_key":"line%1:09:01::","synset_id":"argumentation.n.02"},{"index":10,"sense_key":"line%1:04:00::","synset_id":"occupation.n.01"},{"index":11,"sense_key":"line%1:14:00::","synset_id":"lineage.n.01"},{"index":12,"sense_key":"line%1:06:01::","synset_id":"cable.n.02"},{"index":13,"sense_key":"line%1:06:06::","synset_id":"pipeline.n.02"},{"index":14,"sense_key":"line%1:10:02::","synset_id":"line.n.05"},{"index":15,"sense_key":"line%1:10:04::","synset_id":"note.n.02"},{"index":16,"sense_key":"line%1:04:01::","synset_id":"line.n.16"},{"index":17,"sense_key":"line%1:09:00::","synset_id":"line.n.29"},{"index":18,"sense_key":"line%1:14:02::","synset_id":"course.n.02"},{"index":19,"sense_key":"line%1:21:00::","synset_id":"credit_line.n.01"},{"index":20,"sense_key":"line%1:25:01::","synset_id":"wrinkle.n.01"},{"index":21,"sense_key":"line%1:15:02::","synset_id":"line.n.07"},{"index":22,"sense_key":"line%1:06:03::","synset_id":"line.n.14"},{"index":23,"sense_key":"line%1:10:06::","synset_id":"line.n.27"},{"index":24,"sense_key":"line%1:14:03::","synset_id":"line.n.01"},{"index":25,"sense_key":"line%1:15:00::","synset_id":"line.n.11"},{"index":26,"sense_key":"line%1:06:05::","synset_id":"line.n.23"},{"index":27,"sense_key":"line%1:10:01::","synset_id":"line.n.02"},{"index":28,"sense_key":"line%1:14:01::","synset_id":"line.n.03"},{"index":29,"sense_key":"line%1:25:00::","synset_id":"line.n.04"},{"index":30,"sense_key":"line%1:10:05::","synset_id":"channel.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"line%1:09:00::","call_ids":["senseval3.d001.s044.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"line%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1716,"cached_input_tokens":0,"output_tokens":1027,"reasoning_output_tokens":1006},"cost":{"currency":"USD","total_usd":0.03939,"input_uncached_usd":0.00858,"input_cached_usd":0.0,"output_usd":0.03081,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":21.037802333012223}
{"item_id":"senseval3.d001.s044.t004","gold_sense_keys":["reverse%1:24:00::"],"candidates":[{"index":1,"sense_key":"reverse%1:06:00::","synset_id":"reverse.n.02"},{"index":2,"sense_key":"reverse%1:04:00::","synset_id":"reverse.n.05"},{"index":3,"sense_key":"reverse%1:24:00::","synset_id":"reverse.n.01"},{"index":4,"sense_key":"reverse%1:06:02::","synset_id":"reverse.n.04"},{"index":5,"sense_key":"reverse%1:04:01::","synset_id":"reversion.n.04"},{"index":6,"sense_key":"reverse%1:11:01::","synset_id":"reverse.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"reverse%1:24:00::","call_ids":["senseval3.d001.s044.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"reverse%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":703,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.005825,"input_uncached_usd":0.0035150000000000003,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.671242041978985}
{"item_id":"senseval3.d001.s045.t001","gold_sense_keys":["investigation%1:04:00::"],"candidates":[{"index":1,"sense_key":"investigation%1:09:00::","synset_id":"probe.n.01"},{"index":2,"sense_key":"investigation%1:04:00::","synset_id":"investigation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"investigation%1:04:00::","call_ids":["senseval3.d001.s045.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"investigation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.00513,"input_uncached_usd":0.00267,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8008429589681327}
{"item_id":"senseval3.d001.s045.t003","gold_sense_keys":["hold%2:42:00::"],"candidates":[{"index":1,"sense_key":"hold%2:35:09::","synset_id":"hold.v.31"},{"index":2,"sense_key":"hold%2:41:15::","synset_id":"control.v.02"},{"index":3,"sense_key":"hold%2:35:02::","synset_id":"hold.v.14"},{"index":4,"sense_key":"hold%2:42:01::","synset_id":"hold.v.13"},{"index":5,"sense_key":"hold%2:42:12::","synset_id":"hold.v.26"},{"index":6,"sense_key":"hold%2:42:14::","synset_id":"contain.v.05"},{"index":7,"sense_key":"hold%2:36:00::","synset_id":"hold.v.03"},{"index":8,"sense_key":"hold%2:33:05::","synset_id":"defend.v.03"},{"index":9,"sense_key":"hold%2:32:15::","synset_id":"agree.v.01"},{"index":10,"sense_key":"hold%2:42:07::","synset_id":"hold.v.17"},{"index":11,"sense_key":"hold%2:42:04::","synset_id":"apply.v.02"},{"index":12,"sense_key":"hold%2:32:13::","synset_id":"oblige.v.02"},{"index":13,"sense_key":"hold%2:42:13::","synset_id":"hold.v.11"},{"index":14,"sense_key":"hold%2:42:00::","synset_id":"keep.v.01"},{"index":15,"sense_key":"hold%2:29:04::","synset_id":"hold.v.36"},{"index":16,"sense_key":"hold%2:31:01::","synset_id":"deem.v.01"},{"index":17,"sense_key":"hold%2:40:04::","synset_id":"bear.v.11"},{"index":18,"sense_key":"hold%2:42:05::","synset_id":"accommodate.v.04"},{"index":19,"sense_key":"hold%2:31:10::","synset_id":"hold.v.23"},{"index":20,"sense_key":"hold%2:40:00::","synset_id":"have.v.01"},{"index":21,"sense_key":"hold%2:35:00::","synset_id":"hold.v.02"},{"index":22,"sense_key":"hold%2:35:03::","synset_id":"restrain.v.03"},{"index":23,"sense_key":"hold%2:35:01::","synset_id":"hold.v.10"},{"index":24,"sense_key":"hold%2:33:00::","synset_id":"hold.v.33"},{"index":25,"sense_key":"hold%2:37:00::","synset_id":"harbor.v.01"},{"index":26,"sense_key":"hold%2:42:08::","synset_id":"defy.v.01"},{"index":27,"sense_key":"hold%2:41:13::","synset_id":"hold.v.28"},{"index":28,"sense_key":"hold%2:40:02::","synset_id":"retain.v.03"},{"index":29,"sense_key":"hold%2:31:04::","synset_id":"hold.v.22"},{"index":30,"sense_key":"hold%2:34:00::","synset_id":"carry.v.33"},{"index":31,"sense_key":"hold%2:32:11::","synset_id":"declare.v.04"},{"index":32,"sense_key":"hold%2:42:02::","synset_id":"prevail.v.02"},{"index":33,"sense_key":"hold%2:38:05::","synset_id":"halt.v.01"},{"index":34,"sense_key":"hold%2:41:14::","synset_id":"hold.v.29"},{"index":35,"sense_key":"hold%2:31:08::","synset_id":"hold.v.16"},{"index":36,"sense_key":"hold%2:41:00::","synset_id":"reserve.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"hold%2:42:00::","call_ids":["senseval3.d001.s045.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"hold%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2087,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":135},"cost":{"currency":"USD","total_usd":0.014965000000000003,"input_uncached_usd":0.010435000000000002,"input_cached_usd":0.0,"output_usd":0.00453,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.689116124995053}
{"item_id":"senseval3.d001.s045.t004","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":2,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":3,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":4,"sense_key":"other%3:00:00::","synset_id":"other.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d001.s045.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.00554,"input_uncached_usd":0.00317,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.500288667040877}
{"item_id":"senseval3.d001.s045.t005","gold_sense_keys":["factor%1:11:00::"],"candidates":[{"index":1,"sense_key":"factor%1:23:00::","synset_id":"divisor.n.01"},{"index":2,"sense_key":"factor%1:08:00::","synset_id":"gene.n.01"},{"index":3,"sense_key":"factor%1:23:01::","synset_id":"factor.n.05"},{"index":4,"sense_key":"factor%1:18:00::","synset_id":"agent.n.04"},{"index":5,"sense_key":"factor%1:09:01::","synset_id":"factor.n.06"},{"index":6,"sense_key":"factor%1:09:00::","synset_id":"component.n.01"},{"index":7,"sense_key":"factor%1:11:00::","synset_id":"factor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"factor%1:09:01::","call_ids":["senseval3.d001.s045.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"factor%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":296,"reasoning_output_tokens":280},"cost":{"currency":"USD","total_usd":0.012709999999999999,"input_uncached_usd":0.0038300000000000005,"input_cached_usd":0.0,"output_usd":0.00888,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.736286167055368}
{"item_id":"senseval3.d001.s045.t006","gold_sense_keys":["constant%5:00:00:invariable:00"],"candidates":[{"index":1,"sense_key":"constant%5:00:00:invariable:00","synset_id":"changeless.s.01"},{"index":2,"sense_key":"constant%5:00:00:continuous:01","synset_id":"ceaseless.s.01"},{"index":3,"sense_key":"constant%3:00:00::","synset_id":"constant.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"constant%5:00:00:invariable:00","call_ids":["senseval3.d001.s045.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"constant%5:00:00:invariable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.0055049999999999995,"input_uncached_usd":0.003075,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8330324579728767}
{"item_id":"senseval3.d001.s045.t007","gold_sense_keys":["difference%1:23:00::"],"candidates":[{"index":1,"sense_key":"difference%1:24:00::","synset_id":"difference.n.04"},{"index":2,"sense_key":"difference%1:11:00::","synset_id":"deviation.n.01"},{"index":3,"sense_key":"difference%1:23:00::","synset_id":"remainder.n.03"},{"index":4,"sense_key":"difference%1:07:00::","synset_id":"difference.n.01"},{"index":5,"sense_key":"difference%1:10:00::","synset_id":"dispute.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"difference%1:23:00::","call_ids":["senseval3.d001.s045.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"difference%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.005955,"input_uncached_usd":0.0033450000000000003,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2990073330001906}
{"item_id":"senseval3.d001.s045.t008","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":4,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":5,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":6,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"state%1:15:01::","call_ids":["senseval3.d001.s045.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"state%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":212,"reasoning_output_tokens":196},"cost":{"currency":"USD","total_usd":0.010625,"input_uncached_usd":0.0042650000000000006,"input_cached_usd":0.0,"output_usd":0.00636,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.060519083053805}
{"item_id":"senseval3.d001.s045.t009","gold_sense_keys":["major%3:00:01::","major%3:00:06::"],"candidates":[{"index":1,"sense_key":"major%3:00:01::","synset_id":"major.a.03"},{"index":2,"sense_key":"major%3:00:07::","synset_id":"major.a.06"},{"index":3,"sense_key":"major%3:00:03::","synset_id":"major.a.07"},{"index":4,"sense_key":"major%3:00:02::","synset_id":"major.a.02"},{"index":5,"sense_key":"major%3:00:05::","synset_id":"major.a.04"},{"index":6,"sense_key":"major%3:00:06::","synset_id":"major.a.01"},{"index":7,"sense_key":"major%5:00:00:senior:00","synset_id":"major.s.01"},{"index":8,"sense_key":"major%3:00:04::","synset_id":"major.a.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"major%3:00:06::","call_ids":["senseval3.d001.s045.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"major%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":510,"reasoning_output_tokens":494},"cost":{"currency":"USD","total_usd":0.018915,"input_uncached_usd":0.003615,"input_cached_usd":0.0,"output_usd":0.015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.854227166972123}
{"item_id":"senseval3.d001.s045.t010","gold_sense_keys":["party%1:14:01::"],"candidates":[{"index":1,"sense_key":"party%1:14:00::","synset_id":"party.n.02"},{"index":2,"sense_key":"party%1:11:00::","synset_id":"party.n.04"},{"index":3,"sense_key":"party%1:18:00::","synset_id":"party.n.05"},{"index":4,"sense_key":"party%1:14:02::","synset_id":"party.n.03"},{"index":5,"sense_key":"party%1:14:01::","synset_id":"party.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"party%1:14:01::","call_ids":["senseval3.d001.s045.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"party%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.004985,"input_uncached_usd":0.003305,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4607187079964206}
{"item_id":"senseval3.d001.s045.t011","gold_sense_keys":["vote%1:09:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":2,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":3,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"},{"index":4,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":5,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"vote%1:04:00::","call_ids":["senseval3.d001.s045.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"vote%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":694,"cached_input_tokens":0,"output_tokens":944,"reasoning_output_tokens":923},"cost":{"currency":"USD","total_usd":0.03179,"input_uncached_usd":0.0034700000000000004,"input_cached_usd":0.0,"output_usd":0.02832,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":18.57583754195366}
{"item_id":"senseval3.d001.s045.t012","gold_sense_keys":["go%2:42:06::"],"candidates":[{"index":1,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":2,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":3,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":4,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":5,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":6,"sense_key":"go%2:38:01::","synset_id":"go.v.03"},{"index":7,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":8,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":9,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":10,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":11,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"},{"index":12,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":13,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"},{"index":14,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":15,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":16,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":17,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":18,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":19,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"},{"index":20,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":21,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":22,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":23,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":24,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"},{"index":25,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":26,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":27,"sense_key":"go%2:30:08::","synset_id":"go.v.09"},{"index":28,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":29,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":30,"sense_key":"go%2:42:08::","synset_id":"go.v.25"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":23,"chosen_sense_key":"go%2:42:06::","call_ids":["senseval3.d001.s045.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":23,"predicted_sense_key":"go%2:42:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1858,"cached_input_tokens":0,"output_tokens":332,"reasoning_output_tokens":316},"cost":{"currency":"USD","total_usd":0.019250000000000003,"input_uncached_usd":0.009290000000000001,"input_cached_usd":0.0,"output_usd":0.00996,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.998766999924555}
{"item_id":"senseval3.d001.s045.t013","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s045.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0046,"input_uncached_usd":0.0029200000000000003,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6069996668957174}
{"item_id":"senseval3.d001.s045.t015","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"},{"index":2,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s045.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.004475,"input_uncached_usd":0.002645,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7544218329712749}
{"item_id":"senseval3.d001.s045.t016","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"},{"index":2,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s045.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.00502,"input_uncached_usd":0.0029200000000000003,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.043058666982688}
{"item_id":"senseval3.d001.s045.t017","gold_sense_keys":["share%1:04:00::"],"candidates":[{"index":1,"sense_key":"share%1:21:01::","synset_id":"share.n.02"},{"index":2,"sense_key":"share%1:04:00::","synset_id":"parcel.n.02"},{"index":3,"sense_key":"share%1:04:01::","synset_id":"contribution.n.01"},{"index":4,"sense_key":"share%1:06:00::","synset_id":"plowshare.n.01"},{"index":5,"sense_key":"share%1:21:00::","synset_id":"share.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"share%1:21:00::","call_ids":["senseval3.d001.s045.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"share%1:21:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":705,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.034905,"input_uncached_usd":0.0035250000000000004,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":19.516566417063586}
{"item_id":"senseval3.d001.s045.t019","gold_sense_keys":["house%1:14:02::"],"candidates":[{"index":1,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":2,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":3,"sense_key":"house%1:06:01::","synset_id":"house.n.12"},{"index":4,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":5,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":9,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":10,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":11,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":12,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"house%1:14:02::","call_ids":["senseval3.d001.s045.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"house%1:14:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":934,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.00608,"input_uncached_usd":0.0046700000000000005,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6300560419913381}
{"item_id":"senseval3.d001.s045.t020","gold_sense_keys":["positive%5:00:00:plus:00"],"candidates":[{"index":1,"sense_key":"positive%5:00:00:plus:00","synset_id":"positive.s.05"},{"index":2,"sense_key":"positive%5:00:00:confident:00","synset_id":"cocksure.s.01"},{"index":3,"sense_key":"positive%3:00:01::","synset_id":"positive.a.01"},{"index":4,"sense_key":"positive%5:00:00:advantageous:00","synset_id":"plus.s.01"},{"index":5,"sense_key":"positive%5:00:00:formal:01","synset_id":"positive.s.03"},{"index":6,"sense_key":"positive%3:01:00::","synset_id":"positivist.a.01"},{"index":7,"sense_key":"positive%3:00:04::","synset_id":"positive.a.04"},{"index":8,"sense_key":"positive%5:00:00:undeniable:00","synset_id":"incontrovertible.s.01"},{"index":9,"sense_key":"positive%3:00:05::","synset_id":"positive.a.08"},{"index":10,"sense_key":"positive%5:00:00:charged:00","synset_id":"positive.s.06"},{"index":11,"sense_key":"positive%5:00:00:certain:02","synset_id":"convinced.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"positive%3:00:05::","call_ids":["senseval3.d001.s045.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"positive%3:00:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":942,"cached_input_tokens":0,"output_tokens":866,"reasoning_output_tokens":845},"cost":{"currency":"USD","total_usd":0.030690000000000002,"input_uncached_usd":0.004710000000000001,"input_cached_usd":0.0,"output_usd":0.02598,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.495319458073936}
{"item_id":"senseval3.d001.s045.t021","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":2,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":3,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":4,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":5,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":6,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"},{"index":7,"sense_key":"function%1:24:01::","synset_id":"function.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"function%1:24:01::","call_ids":["senseval3.d001.s045.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"function%1:24:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":798,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":94},"cost":{"currency":"USD","total_usd":0.0072900000000000005,"input_uncached_usd":0.0039900000000000005,"input_cached_usd":0.0,"output_usd":0.0033,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.204429707955569}
{"item_id":"senseval3.d001.s045.t022","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":2,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":3,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":4,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":5,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":6,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"},{"index":7,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":8,"sense_key":"state%1:15:01::","synset_id":"state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"state%1:15:01::","call_ids":["senseval3.d001.s045.t022__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"state%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":90},"cost":{"currency":"USD","total_usd":0.00744,"input_uncached_usd":0.004260000000000001,"input_cached_usd":0.0,"output_usd":0.00318,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9525947499787435}
{"item_id":"senseval3.d001.s046.t001","gold_sense_keys":["federal%5:00:00:national:01"],"candidates":[{"index":1,"sense_key":"federal%3:00:00::","synset_id":"federal.a.04"},{"index":2,"sense_key":"federal%3:01:02::","synset_id":"federal.a.02"},{"index":3,"sense_key":"federal%5:00:00:national:01","synset_id":"federal.s.01"},{"index":4,"sense_key":"federal%5:00:00:northern:02","synset_id":"union.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"federal%5:00:00:national:01","call_ids":["senseval3.d001.s046.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"federal%5:00:00:national:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018885,"input_uncached_usd":0.0030450000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.948012667009607}
{"item_id":"senseval3.d001.s046.t004","gold_sense_keys":["level%1:09:00::"],"candidates":[{"index":1,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":2,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":3,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"},{"index":4,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":8,"sense_key":"level%1:07:01::","synset_id":"level.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"level%1:26:01::","call_ids":["senseval3.d001.s046.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"level%1:26:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":172,"reasoning_output_tokens":156},"cost":{"currency":"USD","total_usd":0.00897,"input_uncached_usd":0.0038100000000000005,"input_cached_usd":0.0,"output_usd":0.0051600000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.906980959000066}
{"item_id":"senseval3.d001.s046.t005","gold_sense_keys":["part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":2,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":3,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":4,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":5,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":6,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":7,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":8,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":9,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":10,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":11,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"part%1:24:00::","call_ids":["senseval3.d001.s046.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"part%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":993,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020805,"input_uncached_usd":0.004965000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.076099000056274}
{"item_id":"senseval3.d001.s046.t006","gold_sense_keys":["irrational%3:00:00::"],"candidates":[{"index":1,"sense_key":"irrational%3:01:00::","synset_id":"irrational.a.02"},{"index":2,"sense_key":"irrational%3:00:00::","synset_id":"irrational.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"irrational%3:00:00::","call_ids":["senseval3.d001.s046.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"irrational%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.004445000000000001,"input_uncached_usd":0.0023450000000000003,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6551514170132577}
{"item_id":"senseval3.d001.s046.t007","gold_sense_keys":["behavior%1:04:00::"],"candidates":[{"index":1,"sense_key":"behavior%1:04:01::","synset_id":"behavior.n.04"},{"index":2,"sense_key":"behavior%1:07:00::","synset_id":"demeanor.n.01"},{"index":3,"sense_key":"behavior%1:04:00::","synset_id":"behavior.n.01"},{"index":4,"sense_key":"behavior%1:26:00::","synset_id":"behavior.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"behavior%1:04:01::","call_ids":["senseval3.d001.s046.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"behavior%1:04:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":5698,"reasoning_output_tokens":5621},"cost":{"currency":"USD","total_usd":0.17378500000000002,"input_uncached_usd":0.0028450000000000003,"input_cached_usd":0.0,"output_usd":0.17094,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":113.29863045900129}
{"item_id":"senseval3.d001.s046.t009","gold_sense_keys":["display%2:39:00::"],"candidates":[{"index":1,"sense_key":"display%2:41:00::","synset_id":"display.v.02"},{"index":2,"sense_key":"display%2:39:00::","synset_id":"expose.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"display%2:39:00::","call_ids":["senseval3.d001.s046.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"display%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.004175,"input_uncached_usd":0.002435,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6581271250033751}
{"item_id":"senseval3.d001.s046.t012","gold_sense_keys":["rational%3:00:00::"],"candidates":[{"index":1,"sense_key":"rational%3:00:00::","synset_id":"rational.a.01"},{"index":2,"sense_key":"rational%5:00:00:intellectual:00","synset_id":"rational.s.02"},{"index":3,"sense_key":"rational%5:00:00:mental:00","synset_id":"intellectual.s.01"},{"index":4,"sense_key":"rational%3:01:00::","synset_id":"rational.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rational%3:00:00::","call_ids":["senseval3.d001.s046.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rational%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004845,"input_uncached_usd":0.0028350000000000003,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.756784249912016}
{"item_id":"senseval3.d002.s000.t000","gold_sense_keys":["long%3:00:02::"],"candidates":[{"index":1,"sense_key":"long%3:00:00::","synset_id":"retentive.a.01"},{"index":2,"sense_key":"long%5:00:00:tall:00","synset_id":"long.s.01"},{"index":3,"sense_key":"long%3:00:05::","synset_id":"long.a.05"},{"index":4,"sense_key":"long%3:00:01::","synset_id":"long.a.02"},{"index":5,"sense_key":"long%3:00:02::","synset_id":"long.a.01"},{"index":6,"sense_key":"long%5:00:00:provident:00","synset_id":"farseeing.s.02"},{"index":7,"sense_key":"long%3:00:04::","synset_id":"long.a.06"},{"index":8,"sense_key":"long%5:00:00:abundant:00","synset_id":"long.s.04"},{"index":9,"sense_key":"long%5:00:00:unsound:00","synset_id":"long.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"long%3:00:02::","call_ids":["senseval3.d002.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"long%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004750000000000001,"input_uncached_usd":0.0029800000000000004,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9371703749056906}
{"item_id":"senseval3.d002.s000.t002","gold_sense_keys":["night%1:28:00::"],"candidates":[{"index":1,"sense_key":"night%1:18:00::","synset_id":"nox.n.01"},{"index":2,"sense_key":"night%1:26:00::","synset_id":"night.n.05"},{"index":3,"sense_key":"night%1:28:01::","synset_id":"night.n.02"},{"index":4,"sense_key":"night%1:28:00::","synset_id":"night.n.01"},{"index":5,"sense_key":"night%1:28:02::","synset_id":"night.n.04"},{"index":6,"sense_key":"night%1:28:04::","synset_id":"night.n.03"},{"index":7,"sense_key":"night%1:28:05::","synset_id":"night.n.06"},{"index":8,"sense_key":"night%1:28:03::","synset_id":"night.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"night%1:28:00::","call_ids":["senseval3.d002.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"night%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":2910,"reasoning_output_tokens":2865},"cost":{"currency":"USD","total_usd":0.08947000000000001,"input_uncached_usd":0.00217,"input_cached_usd":0.0,"output_usd":0.0873,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":45.44349383295048}
{"item_id":"senseval3.d002.s000.t003","gold_sense_keys":["devastate%2:30:00::"],"candidates":[{"index":1,"sense_key":"devastate%2:30:00::","synset_id":"lay_waste_to.v.01"},{"index":2,"sense_key":"devastate%2:30:01::","synset_id":"devastate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"devastate%2:30:00::","call_ids":["senseval3.d002.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"devastate%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":278,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.00334,"input_uncached_usd":0.0013900000000000002,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7892166669480503}
{"item_id":"senseval3.d002.s000.t004","gold_sense_keys":["earthquake%1:11:00::"],"candidates":[{"index":1,"sense_key":"earthquake%1:26:00::","synset_id":"earthquake.n.02"},{"index":2,"sense_key":"earthquake%1:11:00::","synset_id":"earthquake.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"earthquake%1:11:00::","call_ids":["senseval3.d002.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"earthquake%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":269,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.0029050000000000005,"input_uncached_usd":0.001345,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8505848749773577}
{"item_id":"senseval3.d002.s000.t005","gold_sense_keys":["resident%1:18:00::"],"candidates":[{"index":1,"sense_key":"resident%1:18:01::","synset_id":"house_physician.n.01"},{"index":2,"sense_key":"resident%1:18:00::","synset_id":"resident.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"resident%1:18:00::","call_ids":["senseval3.d002.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"resident%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":289,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.003125,"input_uncached_usd":0.001445,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.385371541022323}
{"item_id":"senseval3.d002.s000.t006","gold_sense_keys":["search%2:35:00::","search%2:39:00::"],"candidates":[{"index":1,"sense_key":"search%2:35:01::","synset_id":"search.v.04"},{"index":2,"sense_key":"search%2:31:00::","synset_id":"research.v.02"},{"index":3,"sense_key":"search%2:39:00::","synset_id":"search.v.02"},{"index":4,"sense_key":"search%2:35:00::","synset_id":"search.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"search%2:35:00::","call_ids":["senseval3.d002.s000.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"search%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":381,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017745,"input_uncached_usd":0.0019050000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.810191833996214}
{"item_id":"senseval3.d002.s000.t007","gold_sense_keys":["comfort%1:04:00::"],"candidates":[{"index":1,"sense_key":"comfort%1:26:01::","synset_id":"ease.n.02"},{"index":2,"sense_key":"comfort%1:06:00::","synset_id":"quilt.n.01"},{"index":3,"sense_key":"comfort%1:04:01::","synset_id":"comfort.n.07"},{"index":4,"sense_key":"comfort%1:26:00::","synset_id":"comfort.n.01"},{"index":5,"sense_key":"comfort%1:26:02::","synset_id":"comfort.n.05"},{"index":6,"sense_key":"comfort%1:04:00::","synset_id":"consolation.n.02"},{"index":7,"sense_key":"comfort%1:12:00::","synset_id":"comfort.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"comfort%1:04:00::","call_ids":["senseval3.d002.s000.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"comfort%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":741,"reasoning_output_tokens":720},"cost":{"currency":"USD","total_usd":0.024655,"input_uncached_usd":0.002425,"input_cached_usd":0.0,"output_usd":0.02223,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.303712417022325}
{"item_id":"senseval3.d002.s000.t008","gold_sense_keys":["solace%1:12:00::","solace%1:26:00::"],"candidates":[{"index":1,"sense_key":"solace%1:26:00::","synset_id":"solace.n.02"},{"index":2,"sense_key":"solace%1:04:00::","synset_id":"consolation.n.02"},{"index":3,"sense_key":"solace%1:12:00::","synset_id":"consolation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"solace%1:26:00::","call_ids":["senseval3.d002.s000.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"solace%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":308,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01735,"input_uncached_usd":0.0015400000000000001,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.96439450001344}
{"item_id":"senseval3.d002.s001.t001","gold_sense_keys":["screen%1:06:05::"],"candidates":[{"index":1,"sense_key":"screen%1:06:07::","synset_id":"screen_door.n.01"},{"index":2,"sense_key":"screen%1:06:00::","synset_id":"screen.n.05"},{"index":3,"sense_key":"screen%1:06:05::","synset_id":"screen.n.03"},{"index":4,"sense_key":"screen%1:06:02::","synset_id":"sieve.n.01"},{"index":5,"sense_key":"screen%1:06:04::","synset_id":"blind.n.03"},{"index":6,"sense_key":"screen%1:14:00::","synset_id":"filmdom.n.01"},{"index":7,"sense_key":"screen%1:06:03::","synset_id":"screen.n.09"},{"index":8,"sense_key":"screen%1:06:06::","synset_id":"screen.n.01"},{"index":9,"sense_key":"screen%1:06:01::","synset_id":"screen.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"screen%1:06:05::","call_ids":["senseval3.d002.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"screen%1:06:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.00441,"input_uncached_usd":0.00303,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3849853749852628}
{"item_id":"senseval3.d002.s002.t001","gold_sense_keys":["make%2:38:10::"],"candidates":[{"index":1,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":2,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":3,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":4,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":5,"sense_key":"make%2:29:01::","synset_id":"make.v.49"},{"index":6,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":7,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":8,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":9,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":10,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":11,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":12,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":13,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":14,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":15,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":16,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":17,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":18,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":19,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":20,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":21,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":22,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":23,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":24,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":25,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":26,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":27,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":28,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":29,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":30,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":31,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":32,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":33,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":34,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":35,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":36,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":37,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":38,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":39,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":40,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":41,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":42,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":43,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":44,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":45,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":46,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":47,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":48,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":49,"sense_key":"make%2:42:05::","synset_id":"form.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":48,"chosen_sense_key":"make%2:38:10::","call_ids":["senseval3.d002.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":48,"predicted_sense_key":"make%2:38:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2212,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.013250000000000001,"input_uncached_usd":0.01106,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.011225500027649}
{"item_id":"senseval3.d002.s002.t003","gold_sense_keys":["computer%1:06:00::"],"candidates":[{"index":1,"sense_key":"computer%1:18:00::","synset_id":"calculator.n.01"},{"index":2,"sense_key":"computer%1:06:00::","synset_id":"computer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"computer%1:06:00::","call_ids":["senseval3.d002.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"computer%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":337,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.003215,"input_uncached_usd":0.001685,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9206917920382693}
{"item_id":"senseval3.d002.s002.t007","gold_sense_keys":["link%2:35:00::"],"candidates":[{"index":1,"sense_key":"link%2:31:00::","synset_id":"associate.v.01"},{"index":2,"sense_key":"link%2:42:01::","synset_id":"connect.v.03"},{"index":3,"sense_key":"link%2:35:00::","synset_id":"connect.v.01"},{"index":4,"sense_key":"link%2:35:02::","synset_id":"yoke.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"link%2:35:00::","call_ids":["senseval3.d002.s002.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"link%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":121},"cost":{"currency":"USD","total_usd":0.006405,"input_uncached_usd":0.002295,"input_cached_usd":0.0,"output_usd":0.00411,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6142535000108182}
{"item_id":"senseval3.d002.s002.t008","gold_sense_keys":["computer%1:06:00::"],"candidates":[{"index":1,"sense_key":"computer%1:18:00::","synset_id":"calculator.n.01"},{"index":2,"sense_key":"computer%1:06:00::","synset_id":"computer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"computer%1:06:00::","call_ids":["senseval3.d002.s002.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"computer%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":337,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.003275,"input_uncached_usd":0.001685,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.629104332998395}
{"item_id":"senseval3.d002.s003.t000","gold_sense_keys":["vivid%5:00:00:clear:00","vivid%5:00:00:realistic:00"],"candidates":[{"index":1,"sense_key":"vivid%5:00:00:realistic:00","synset_id":"graphic.s.03"},{"index":2,"sense_key":"vivid%5:00:00:saturated:03","synset_id":"intense.s.02"},{"index":3,"sense_key":"vivid%5:00:00:clear:00","synset_id":"vivid.s.02"},{"index":4,"sense_key":"vivid%5:00:00:colorful:00","synset_id":"bright.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vivid%5:00:00:realistic:00","call_ids":["senseval3.d002.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vivid%5:00:00:realistic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":438,"cached_input_tokens":0,"output_tokens":245,"reasoning_output_tokens":229},"cost":{"currency":"USD","total_usd":0.00954,"input_uncached_usd":0.00219,"input_cached_usd":0.0,"output_usd":0.00735,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9922008749563247}
{"item_id":"senseval3.d002.s003.t002","gold_sense_keys":["come%2:38:02::"],"candidates":[{"index":1,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":2,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":3,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":4,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":5,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":6,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":7,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":8,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":9,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":10,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":11,"sense_key":"come%2:31:13::","synset_id":"come.v.21"},{"index":12,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":13,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":14,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":15,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":16,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":17,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":18,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":19,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":20,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":21,"sense_key":"come%2:42:02::","synset_id":"do.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"come%2:38:02::","call_ids":["senseval3.d002.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"come%2:38:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1146,"cached_input_tokens":0,"output_tokens":333,"reasoning_output_tokens":317},"cost":{"currency":"USD","total_usd":0.01572,"input_uncached_usd":0.005730000000000001,"input_cached_usd":0.0,"output_usd":0.00999,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.144077833974734}
{"item_id":"senseval3.d002.s003.t004","gold_sense_keys":["electronic%3:01:00::"],"candidates":[{"index":1,"sense_key":"electronic%3:01:01::","synset_id":"electronic.a.02"},{"index":2,"sense_key":"electronic%3:01:00::","synset_id":"electronic.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"electronic%3:01:00::","call_ids":["senseval3.d002.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"electronic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":347,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.003985000000000001,"input_uncached_usd":0.0017350000000000002,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4104035830823705}
{"item_id":"senseval3.d002.s004.t001","gold_sense_keys":["subscriber%1:18:01::"],"candidates":[{"index":1,"sense_key":"subscriber%1:18:01::","synset_id":"subscriber.n.02"},{"index":2,"sense_key":"subscriber%1:18:02::","synset_id":"subscriber.n.01"},{"index":3,"sense_key":"subscriber%1:18:00::","synset_id":"subscriber.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"subscriber%1:18:01::","call_ids":["senseval3.d002.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"subscriber%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.004325000000000001,"input_uncached_usd":0.0020450000000000004,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.047409249935299}
{"item_id":"senseval3.d002.s004.t002","gold_sense_keys":["live%2:42:08::"],"candidates":[{"index":1,"sense_key":"live%2:31:00::","synset_id":"know.v.05"},{"index":2,"sense_key":"live%2:42:07::","synset_id":"live.v.07"},{"index":3,"sense_key":"live%2:42:06::","synset_id":"live.v.02"},{"index":4,"sense_key":"live%2:42:00::","synset_id":"be.v.11"},{"index":5,"sense_key":"live%2:42:01::","synset_id":"exist.v.02"},{"index":6,"sense_key":"live%2:42:08::","synset_id":"populate.v.01"},{"index":7,"sense_key":"live%2:42:04::","synset_id":"survive.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"live%2:42:08::","call_ids":["senseval3.d002.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"live%2:42:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":650,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.0052,"input_uncached_usd":0.0032500000000000003,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8040548339486122}
{"item_id":"senseval3.d002.s005.t002","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"},{"index":2,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"},{"index":3,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"},{"index":4,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"hour%1:28:00::","call_ids":["senseval3.d002.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.004065,"input_uncached_usd":0.0023250000000000002,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4767080000601709}
{"item_id":"senseval3.d002.s005.t003","gold_sense_keys":["come_up%2:35:00::"],"candidates":[{"index":1,"sense_key":"come_up%2:38:03::","synset_id":"rise.v.01"},{"index":2,"sense_key":"come_up%2:42:00::","synset_id":"arise.v.02"},{"index":3,"sense_key":"come_up%2:40:00::","synset_id":"line_up.v.02"},{"index":4,"sense_key":"come_up%2:42:01::","synset_id":"come_up.v.07"},{"index":5,"sense_key":"come_up%2:35:01::","synset_id":"muster.v.01"},{"index":6,"sense_key":"come_up%2:38:01::","synset_id":"surface.v.01"},{"index":7,"sense_key":"come_up%2:36:03::","synset_id":"come_up.v.01"},{"index":8,"sense_key":"come_up%2:35:00::","synset_id":"go_on.v.05"},{"index":9,"sense_key":"come_up%2:35:02::","synset_id":"scrape.v.05"},{"index":10,"sense_key":"come_up%2:38:00::","synset_id":"rise.v.16"},{"index":11,"sense_key":"come_up%2:30:00::","synset_id":"arise.v.04"},{"index":12,"sense_key":"come_up%2:38:02::","synset_id":"come.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"come_up%2:35:00::","call_ids":["senseval3.d002.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"come_up%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":836,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.006790000000000001,"input_uncached_usd":0.0041800000000000006,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.262679083039984}
{"item_id":"senseval3.d002.s005.t004","gold_sense_keys":["teem%2:42:00::"],"candidates":[{"index":1,"sense_key":"teem%2:42:00::","synset_id":"teem.v.01"},{"index":2,"sense_key":"teem%2:38:00::","synset_id":"pour.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teem%2:42:00::","call_ids":["senseval3.d002.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teem%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":413,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.004765,"input_uncached_usd":0.002065,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1550592499552295}
{"item_id":"senseval3.d002.s005.t005","gold_sense_keys":["emotional%3:00:00::"],"candidates":[{"index":1,"sense_key":"emotional%3:00:02::","synset_id":"emotional.a.01"},{"index":2,"sense_key":"emotional%3:00:00::","synset_id":"emotional.a.02"},{"index":3,"sense_key":"emotional%5:00:00:agitated:00","synset_id":"aroused.s.06"},{"index":4,"sense_key":"emotional%3:01:00::","synset_id":"emotional.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"emotional%3:00:00::","call_ids":["senseval3.d002.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"emotional%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":345,"reasoning_output_tokens":329},"cost":{"currency":"USD","total_usd":0.012695,"input_uncached_usd":0.0023450000000000003,"input_cached_usd":0.0,"output_usd":0.01035,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.871800333028659}
{"item_id":"senseval3.d002.s005.t007","gold_sense_keys":["report%1:10:00::"],"candidates":[{"index":1,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"},{"index":2,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":3,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":7,"sense_key":"report%1:10:01::","synset_id":"report.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"report%1:10:00::","call_ids":["senseval3.d002.s005.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"report%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":605,"reasoning_output_tokens":583},"cost":{"currency":"USD","total_usd":0.021315000000000004,"input_uncached_usd":0.0031650000000000003,"input_cached_usd":0.0,"output_usd":0.01815,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.586758499965072}
{"item_id":"senseval3.d002.s006.t001","gold_sense_keys":["electronic%3:01:00::"],"candidates":[{"index":1,"sense_key":"electronic%3:01:00::","synset_id":"electronic.a.01"},{"index":2,"sense_key":"electronic%3:01:01::","synset_id":"electronic.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"electronic%3:01:00::","call_ids":["senseval3.d002.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"electronic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":381,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0037950000000000006,"input_uncached_usd":0.0019050000000000002,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7900284579955041}
{"item_id":"senseval3.d002.s006.t002","gold_sense_keys":["traffic%1:10:00::"],"candidates":[{"index":1,"sense_key":"traffic%1:04:02::","synset_id":"dealings.n.01"},{"index":2,"sense_key":"traffic%1:10:00::","synset_id":"traffic.n.03"},{"index":3,"sense_key":"traffic%1:04:01::","synset_id":"traffic.n.02"},{"index":4,"sense_key":"traffic%1:14:00::","synset_id":"traffic.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"traffic%1:10:00::","call_ids":["senseval3.d002.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"traffic%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":453,"reasoning_output_tokens":437},"cost":{"currency":"USD","total_usd":0.01587,"input_uncached_usd":0.0022800000000000003,"input_cached_usd":0.0,"output_usd":0.01359,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.660498833050951}
{"item_id":"senseval3.d002.s007.t000","gold_sense_keys":["time%1:28:03::"],"candidates":[{"index":1,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":2,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":3,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":4,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":5,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":6,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":9,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":10,"sense_key":"time%1:03:00::","synset_id":"time.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"time%1:28:03::","call_ids":["senseval3.d002.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"time%1:28:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":709,"cached_input_tokens":0,"output_tokens":183,"reasoning_output_tokens":167},"cost":{"currency":"USD","total_usd":0.009035000000000001,"input_uncached_usd":0.0035450000000000004,"input_cached_usd":0.0,"output_usd":0.00549,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.879108375054784}
{"item_id":"senseval3.d002.s007.t001","gold_sense_keys":["time%1:28:03::"],"candidates":[{"index":1,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":2,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":3,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":4,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"},{"index":5,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":6,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":7,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":8,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":9,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":10,"sense_key":"time%1:11:01::","synset_id":"time.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"time%1:28:03::","call_ids":["senseval3.d002.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"time%1:28:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":708,"cached_input_tokens":0,"output_tokens":264,"reasoning_output_tokens":248},"cost":{"currency":"USD","total_usd":0.01146,"input_uncached_usd":0.00354,"input_cached_usd":0.0,"output_usd":0.00792,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.724581749993376}
{"item_id":"senseval3.d002.s007.t004","gold_sense_keys":["subscriber%1:18:01::"],"candidates":[{"index":1,"sense_key":"subscriber%1:18:02::","synset_id":"subscriber.n.01"},{"index":2,"sense_key":"subscriber%1:18:01::","synset_id":"subscriber.n.02"},{"index":3,"sense_key":"subscriber%1:18:00::","synset_id":"subscriber.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"subscriber%1:18:01::","call_ids":["senseval3.d002.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"subscriber%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.00468,"input_uncached_usd":0.0020700000000000002,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2142008749069646}
{"item_id":"senseval3.d002.s007.t005","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":2,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":3,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":4,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":5,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":6,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval3.d002.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00461,"input_uncached_usd":0.0029300000000000003,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8248841660097241}
{"item_id":"senseval3.d002.s007.t006","gold_sense_keys":["identify%2:32:00::"],"candidates":[{"index":1,"sense_key":"identify%2:31:01::","synset_id":"identify.v.05"},{"index":2,"sense_key":"identify%2:31:02::","synset_id":"identify.v.06"},{"index":3,"sense_key":"identify%2:31:00::","synset_id":"identify.v.01"},{"index":4,"sense_key":"identify%2:31:04::","synset_id":"identify.v.03"},{"index":5,"sense_key":"identify%2:32:00::","synset_id":"name.v.02"},{"index":6,"sense_key":"identify%2:31:03::","synset_id":"identify.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"identify%2:32:00::","call_ids":["senseval3.d002.s007.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"identify%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":561,"cached_input_tokens":0,"output_tokens":196,"reasoning_output_tokens":180},"cost":{"currency":"USD","total_usd":0.008685000000000002,"input_uncached_usd":0.002805,"input_cached_usd":0.0,"output_usd":0.00588,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6933259159559384}
{"item_id":"senseval3.d002.s011.t000","gold_sense_keys":["avenue%1:06:00::"],"candidates":[{"index":1,"sense_key":"avenue%1:04:00::","synset_id":"avenue.n.01"},{"index":2,"sense_key":"avenue%1:06:00::","synset_id":"avenue.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"avenue%1:06:00::","call_ids":["senseval3.d002.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"avenue%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":327,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":103},"cost":{"currency":"USD","total_usd":0.0052049999999999996,"input_uncached_usd":0.0016350000000000002,"input_cached_usd":0.0,"output_usd":0.0035700000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5391359579516575}
{"item_id":"senseval3.d002.s011.t002","gold_sense_keys":["floor%1:06:01::"],"candidates":[{"index":1,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"},{"index":2,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"},{"index":3,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":4,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":5,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":6,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"},{"index":7,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"},{"index":8,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":9,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"},{"index":10,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"floor%1:06:01::","call_ids":["senseval3.d002.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"floor%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":656,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.00517,"input_uncached_usd":0.0032800000000000004,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0035233329981565}
{"item_id":"senseval3.d002.s011.t003","gold_sense_keys":["old%3:00:01::"],"candidates":[{"index":1,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"},{"index":2,"sense_key":"old%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"},{"index":3,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":4,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":5,"sense_key":"old%5:00:00:preceding:00","synset_id":"previous.s.01"},{"index":6,"sense_key":"old%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":7,"sense_key":"old%5:00:03:past:00","synset_id":"erstwhile.s.01"},{"index":8,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"old%3:00:01::","call_ids":["senseval3.d002.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"old%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":632,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00508,"input_uncached_usd":0.00316,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.760305125033483}
{"item_id":"senseval3.d002.s011.t004","gold_sense_keys":["building%1:06:00::"],"candidates":[{"index":1,"sense_key":"building%1:04:01::","synset_id":"construction.n.07"},{"index":2,"sense_key":"building%1:04:00::","synset_id":"construction.n.01"},{"index":3,"sense_key":"building%1:06:00::","synset_id":"building.n.01"},{"index":4,"sense_key":"building%1:14:00::","synset_id":"building.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"building%1:06:00::","call_ids":["senseval3.d002.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"building%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.0036750000000000003,"input_uncached_usd":0.0022050000000000004,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9274427499622107}
{"item_id":"senseval3.d002.s011.t005","gold_sense_keys":["heart%1:08:00::"],"candidates":[{"index":1,"sense_key":"heart%1:13:00::","synset_id":"heart.n.08"},{"index":2,"sense_key":"heart%1:09:01::","synset_id":"kernel.n.03"},{"index":3,"sense_key":"heart%1:12:04::","synset_id":"affection.n.01"},{"index":4,"sense_key":"heart%1:07:04::","synset_id":"heart.n.06"},{"index":5,"sense_key":"heart%1:06:00::","synset_id":"heart.n.10"},{"index":6,"sense_key":"heart%1:08:00::","synset_id":"heart.n.02"},{"index":7,"sense_key":"heart%1:25:00::","synset_id":"heart.n.07"},{"index":8,"sense_key":"heart%1:07:01::","synset_id":"heart.n.03"},{"index":9,"sense_key":"heart%1:09:00::","synset_id":"heart.n.01"},{"index":10,"sense_key":"heart%1:15:00::","synset_id":"center.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"heart%1:08:00::","call_ids":["senseval3.d002.s011.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"heart%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.0063300000000000006,"input_uncached_usd":0.00387,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.584354709018953}
{"item_id":"senseval3.d002.s012.t000","gold_sense_keys":["fall%2:38:03::"],"candidates":[{"index":1,"sense_key":"fall%2:35:00::","synset_id":"hang.v.05"},{"index":2,"sense_key":"fall%2:40:13::","synset_id":"fall.v.21"},{"index":3,"sense_key":"fall%2:41:00::","synset_id":"fall.v.13"},{"index":4,"sense_key":"fall%2:41:01::","synset_id":"fall.v.12"},{"index":5,"sense_key":"fall%2:30:00::","synset_id":"fall.v.10"},{"index":6,"sense_key":"fall%2:40:08::","synset_id":"fall.v.20"},{"index":7,"sense_key":"fall%2:30:15::","synset_id":"fall.v.31"},{"index":8,"sense_key":"fall%2:38:15::","synset_id":"fall.v.23"},{"index":9,"sense_key":"fall%2:41:02::","synset_id":"fall.v.06"},{"index":10,"sense_key":"fall%2:42:04::","synset_id":"fall.v.08"},{"index":11,"sense_key":"fall%2:42:00::","synset_id":"fall.v.07"},{"index":12,"sense_key":"fall%2:38:01::","synset_id":"fall.v.15"},{"index":13,"sense_key":"fall%2:41:15::","synset_id":"fall.v.17"},{"index":14,"sense_key":"fall%2:38:00::","synset_id":"descend.v.01"},{"index":15,"sense_key":"fall%2:40:12::","synset_id":"fall.v.18"},{"index":16,"sense_key":"fall%2:40:09::","synset_id":"accrue.v.02"},{"index":17,"sense_key":"fall%2:38:14::","synset_id":"fall.v.24"},{"index":18,"sense_key":"fall%2:30:14::","synset_id":"fall.v.27"},{"index":19,"sense_key":"fall%2:38:11::","synset_id":"fall.v.22"},{"index":20,"sense_key":"fall%2:42:03::","synset_id":"fall.v.04"},{"index":21,"sense_key":"fall%2:42:02::","synset_id":"fall.v.16"},{"index":22,"sense_key":"fall%2:38:03::","synset_id":"fall.v.01"},{"index":23,"sense_key":"fall%2:30:12::","synset_id":"fall.v.30"},{"index":24,"sense_key":"fall%2:40:03::","synset_id":"fall.v.14"},{"index":25,"sense_key":"fall%2:30:10::","synset_id":"fall.v.28"},{"index":26,"sense_key":"fall%2:43:00::","synset_id":"precipitate.v.03"},{"index":27,"sense_key":"fall%2:30:13::","synset_id":"fall.v.32"},{"index":28,"sense_key":"fall%2:30:09::","synset_id":"fall.v.29"},{"index":29,"sense_key":"fall%2:30:06::","synset_id":"decrease.v.01"},{"index":30,"sense_key":"fall%2:30:08::","synset_id":"fall.v.03"},{"index":31,"sense_key":"fall%2:40:05::","synset_id":"fall.v.09"},{"index":32,"sense_key":"fall%2:30:11::","synset_id":"fall.v.26"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"fall%2:38:15::","call_ids":["senseval3.d002.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"fall%2:38:15::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1514,"cached_input_tokens":0,"output_tokens":493,"reasoning_output_tokens":477},"cost":{"currency":"USD","total_usd":0.022359999999999998,"input_uncached_usd":0.00757,"input_cached_usd":0.0,"output_usd":0.014790000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.383877750020474}
{"item_id":"senseval3.d002.s012.t001","gold_sense_keys":["poster%1:10:00::"],"candidates":[{"index":1,"sense_key":"poster%1:10:00::","synset_id":"poster.n.01"},{"index":2,"sense_key":"poster%1:05:00::","synset_id":"post_horse.n.01"},{"index":3,"sense_key":"poster%1:18:00::","synset_id":"bill_poster.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"poster%1:10:00::","call_ids":["senseval3.d002.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"poster%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":381,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.004035,"input_uncached_usd":0.0019050000000000002,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0415009160060436}
{"item_id":"senseval3.d002.s012.t002","gold_sense_keys":["frame%1:06:00::"],"candidates":[{"index":1,"sense_key":"frame%1:06:00::","synset_id":"frame.n.10"},{"index":2,"sense_key":"frame%1:28:00::","synset_id":"inning.n.01"},{"index":3,"sense_key":"frame%1:06:03::","synset_id":"frame.n.02"},{"index":4,"sense_key":"frame%1:06:02::","synset_id":"frame.n.01"},{"index":5,"sense_key":"frame%1:06:01::","synset_id":"skeleton.n.04"},{"index":6,"sense_key":"frame%1:10:00::","synset_id":"frame.n.05"},{"index":7,"sense_key":"frame%1:04:01::","synset_id":"frame.n.11"},{"index":8,"sense_key":"frame%1:09:00::","synset_id":"frame_of_reference.n.02"},{"index":9,"sense_key":"frame%1:10:01::","synset_id":"frame.n.06"},{"index":10,"sense_key":"frame%1:08:01::","synset_id":"human_body.n.01"},{"index":11,"sense_key":"frame%1:08:00::","synset_id":"skeletal_system.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"frame%1:06:00::","call_ids":["senseval3.d002.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"frame%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.005355,"input_uncached_usd":0.003435,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0416874999646097}
{"item_id":"senseval3.d002.s012.t003","gold_sense_keys":["glass%1:27:00::"],"candidates":[{"index":1,"sense_key":"glass%1:27:00::","synset_id":"glass.n.01"},{"index":2,"sense_key":"glass%1:06:01::","synset_id":"methamphetamine.n.01"},{"index":3,"sense_key":"glass%1:06:02::","synset_id":"looking_glass.n.01"},{"index":4,"sense_key":"glass%1:06:03::","synset_id":"field_glass.n.01"},{"index":5,"sense_key":"glass%1:23:00::","synset_id":"glass.n.03"},{"index":6,"sense_key":"glass%1:06:00::","synset_id":"glass.n.02"},{"index":7,"sense_key":"glass%1:06:04::","synset_id":"glass.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"glass%1:27:00::","call_ids":["senseval3.d002.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"glass%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.004645,"input_uncached_usd":0.0024850000000000002,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6728861670708284}
{"item_id":"senseval3.d002.s012.t004","gold_sense_keys":["floor%1:06:00::"],"candidates":[{"index":1,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":2,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"},{"index":3,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"},{"index":4,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"},{"index":5,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"},{"index":6,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":7,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"},{"index":8,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":9,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":10,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"floor%1:06:00::","call_ids":["senseval3.d002.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"floor%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.00575,"input_uncached_usd":0.0032600000000000003,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2276389159960672}
{"item_id":"senseval3.d002.s012.t006","gold_sense_keys":["open%3:00:01::"],"candidates":[{"index":1,"sense_key":"open%5:00:00:susceptible:00","synset_id":"capable.s.01"},{"index":2,"sense_key":"open%5:00:00:vulnerable:00","synset_id":"assailable.s.01"},{"index":3,"sense_key":"open%3:00:02::","synset_id":"open.a.02"},{"index":4,"sense_key":"open%5:00:00:public:00","synset_id":"open.s.02"},{"index":5,"sense_key":"open%5:00:00:unconstricted:00","synset_id":"open.s.10"},{"index":6,"sense_key":"open%5:00:00:unobstructed:00","synset_id":"clear.s.02"},{"index":7,"sense_key":"open%3:00:08::","synset_id":"open.a.05"},{"index":8,"sense_key":"open%5:00:00:available:00","synset_id":"open.s.03"},{"index":9,"sense_key":"open%5:00:00:unsettled:02","synset_id":"open.s.08"},{"index":10,"sense_key":"open%3:00:01::","synset_id":"open.a.01"},{"index":11,"sense_key":"open%3:00:04::","synset_id":"open.a.11"},{"index":12,"sense_key":"open%5:00:00:ingenuous:00","synset_id":"candid.s.03"},{"index":13,"sense_key":"open%5:00:00:unrestricted:00","synset_id":"open.s.04"},{"index":14,"sense_key":"open%3:00:00::","synset_id":"receptive.a.02"},{"index":15,"sense_key":"open%5:00:00:unenclosed:00","synset_id":"open.s.07"},{"index":16,"sense_key":"open%5:00:00:unsealed:01","synset_id":"open.s.09"},{"index":17,"sense_key":"open%5:00:00:nonunion:00","synset_id":"open.s.11"},{"index":18,"sense_key":"open%5:00:00:coarse:00","synset_id":"loose.s.06"},{"index":19,"sense_key":"open%5:00:00:unprotected:00","synset_id":"exposed.s.01"},{"index":20,"sense_key":"open%5:00:00:active:06","synset_id":"open.s.15"},{"index":21,"sense_key":"open%3:00:07::","synset_id":"overt.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"open%3:00:01::","call_ids":["senseval3.d002.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"open%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1099,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":105},"cost":{"currency":"USD","total_usd":0.009125000000000001,"input_uncached_usd":0.005495000000000001,"input_cached_usd":0.0,"output_usd":0.00363,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2213210409972817}
{"item_id":"senseval3.d002.s012.t007","gold_sense_keys":["floor%1:06:00::"],"candidates":[{"index":1,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"},{"index":2,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"},{"index":3,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":4,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":5,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":6,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":7,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"},{"index":8,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"},{"index":9,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"},{"index":10,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"floor%1:06:00::","call_ids":["senseval3.d002.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"floor%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.005810000000000001,"input_uncached_usd":0.0032600000000000003,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8631602079840377}
{"item_id":"senseval3.d002.s015.t000","gold_sense_keys":["favorite%5:00:00:loved:00"],"candidates":[{"index":1,"sense_key":"favorite%5:00:00:loved:00","synset_id":"favored.s.01"},{"index":2,"sense_key":"favorite%5:00:00:popular:00","synset_id":"favorite.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"favorite%5:00:00:loved:00","call_ids":["senseval3.d002.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"favorite%5:00:00:loved:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":364,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.00407,"input_uncached_usd":0.0018200000000000002,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0005067079328}
{"item_id":"senseval3.d002.s015.t001","gold_sense_keys":["wait%2:42:00::"],"candidates":[{"index":1,"sense_key":"wait%2:42:01::","synset_id":"wait.v.02"},{"index":2,"sense_key":"wait%2:31:00::","synset_id":"expect.v.03"},{"index":3,"sense_key":"wait%2:42:00::","synset_id":"wait.v.01"},{"index":4,"sense_key":"wait%2:41:00::","synset_id":"wait.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"wait%2:42:00::","call_ids":["senseval3.d002.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"wait%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.005745,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.00348,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3339576669968665}
{"item_id":"senseval3.d002.s015.t002","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":2,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":3,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":4,"sense_key":"game%1:04:04::","synset_id":"game.n.11"},{"index":5,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":6,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":10,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":11,"sense_key":"game%1:04:01::","synset_id":"game.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"game%1:04:03::","call_ids":["senseval3.d002.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":108},"cost":{"currency":"USD","total_usd":0.0072250000000000005,"input_uncached_usd":0.0035050000000000003,"input_cached_usd":0.0,"output_usd":0.00372,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4764266250422224}
{"item_id":"senseval3.d002.s015.t003","gold_sense_keys":["start%2:42:00::"],"candidates":[{"index":1,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":2,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":3,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":4,"sense_key":"start%2:38:03::","synset_id":"start.v.14"},{"index":5,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":6,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":7,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":8,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":9,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":10,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":11,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":12,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":13,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":14,"sense_key":"start%2:38:01::","synset_id":"start.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"start%2:42:00::","call_ids":["senseval3.d002.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"start%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":903,"cached_input_tokens":0,"output_tokens":608,"reasoning_output_tokens":586},"cost":{"currency":"USD","total_usd":0.022755000000000004,"input_uncached_usd":0.004515000000000001,"input_cached_usd":0.0,"output_usd":0.01824,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.598447000025772}
{"item_id":"senseval3.d002.s016.t000","gold_sense_keys":["feel%2:39:00::"],"candidates":[{"index":1,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":2,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":3,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":4,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":5,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":6,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"},{"index":7,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":8,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":9,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":10,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":11,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":12,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":13,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"feel%2:39:00::","call_ids":["senseval3.d002.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"feel%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.00738,"input_uncached_usd":0.00417,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.096184292109683}
{"item_id":"senseval3.d002.s016.t002","gold_sense_keys":["begin%2:42:00::","begin%2:42:04::"],"candidates":[{"index":1,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":2,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":3,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":4,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":5,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":6,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"},{"index":9,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":10,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"begin%2:42:04::","call_ids":["senseval3.d002.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"begin%2:42:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":797,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019825000000000002,"input_uncached_usd":0.003985000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.6896274170139804}
{"item_id":"senseval3.d002.s016.t003","gold_sense_keys":["glance%2:39:00::"],"candidates":[{"index":1,"sense_key":"glance%2:38:00::","synset_id":"glance.v.02"},{"index":2,"sense_key":"glance%2:39:00::","synset_id":"glance.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"glance%2:39:00::","call_ids":["senseval3.d002.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"glance%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":367,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.003365,"input_uncached_usd":0.001835,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2974531250074506}
{"item_id":"senseval3.d002.s016.t004","gold_sense_keys":["table%1:14:01::"],"candidates":[{"index":1,"sense_key":"table%1:14:01::","synset_id":"table.n.05"},{"index":2,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":3,"sense_key":"table%1:13:01::","synset_id":"board.n.04"},{"index":4,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":5,"sense_key":"table%1:06:02::","synset_id":"table.n.03"},{"index":6,"sense_key":"table%1:14:00::","synset_id":"table.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"table%1:14:01::","call_ids":["senseval3.d002.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"table%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01853,"input_uncached_usd":0.00269,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.840672875056043}
{"item_id":"senseval3.d002.s016.t005","gold_sense_keys":["next%5:00:00:close:01"],"candidates":[{"index":1,"sense_key":"next%5:00:00:succeeding:00","synset_id":"following.s.02"},{"index":2,"sense_key":"next%5:00:00:close:01","synset_id":"adjacent.s.01"},{"index":3,"sense_key":"next%5:00:00:incoming:00","synset_id":"future.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"next%5:00:00:close:01","call_ids":["senseval3.d002.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"next%5:00:00:close:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.00368,"input_uncached_usd":0.0021200000000000004,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4730043329764158}
{"item_id":"senseval3.d002.s016.t007","gold_sense_keys":["guilty%5:00:00:ashamed:00"],"candidates":[{"index":1,"sense_key":"guilty%3:00:00::","synset_id":"guilty.a.01"},{"index":2,"sense_key":"guilty%5:00:00:ashamed:00","synset_id":"guilty.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"guilty%5:00:00:ashamed:00","call_ids":["senseval3.d002.s016.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"guilty%5:00:00:ashamed:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":79},"cost":{"currency":"USD","total_usd":0.00483,"input_uncached_usd":0.00198,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.340396875049919}
{"item_id":"senseval3.d002.s016.t009","gold_sense_keys":["mouth%2:32:01::"],"candidates":[{"index":1,"sense_key":"mouth%2:35:00::","synset_id":"mouth.v.03"},{"index":2,"sense_key":"mouth%2:32:00::","synset_id":"talk.v.02"},{"index":3,"sense_key":"mouth%2:32:01::","synset_id":"mouth.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"mouth%2:32:01::","call_ids":["senseval3.d002.s016.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"mouth%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":165,"reasoning_output_tokens":150},"cost":{"currency":"USD","total_usd":0.0069500000000000004,"input_uncached_usd":0.002,"input_cached_usd":0.0,"output_usd":0.00495,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3020675419829786}
{"item_id":"senseval3.d002.s016.t010","gold_sense_keys":["word%1:10:00::"],"candidates":[{"index":1,"sense_key":"word%1:10:06::","synset_id":"word.n.04"},{"index":2,"sense_key":"word%1:10:03::","synset_id":"word.n.02"},{"index":3,"sense_key":"word%1:23:00::","synset_id":"word.n.07"},{"index":4,"sense_key":"word%1:10:02::","synset_id":"news.n.01"},{"index":5,"sense_key":"word%1:10:07::","synset_id":"bible.n.01"},{"index":6,"sense_key":"word%1:10:05::","synset_id":"password.n.01"},{"index":7,"sense_key":"word%1:10:04::","synset_id":"discussion.n.02"},{"index":8,"sense_key":"word%1:10:01::","synset_id":"parole.n.01"},{"index":9,"sense_key":"word%1:18:00::","synset_id":"son.n.02"},{"index":10,"sense_key":"word%1:10:00::","synset_id":"word.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"word%1:10:00::","call_ids":["senseval3.d002.s016.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"word%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.005755,"input_uncached_usd":0.0035050000000000003,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7099864999763668}
{"item_id":"senseval3.d002.s016.t011","gold_sense_keys":["earthquake%1:11:00::"],"candidates":[{"index":1,"sense_key":"earthquake%1:26:00::","synset_id":"earthquake.n.02"},{"index":2,"sense_key":"earthquake%1:11:00::","synset_id":"earthquake.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"earthquake%1:11:00::","call_ids":["senseval3.d002.s016.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"earthquake%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":376,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.00362,"input_uncached_usd":0.0018800000000000002,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8807664170162752}
{"item_id":"senseval3.d002.s017.t000","gold_sense_keys":["long%3:00:02::"],"candidates":[{"index":1,"sense_key":"long%3:00:01::","synset_id":"long.a.02"},{"index":2,"sense_key":"long%5:00:00:provident:00","synset_id":"farseeing.s.02"},{"index":3,"sense_key":"long%5:00:00:tall:00","synset_id":"long.s.01"},{"index":4,"sense_key":"long%3:00:02::","synset_id":"long.a.01"},{"index":5,"sense_key":"long%3:00:05::","synset_id":"long.a.05"},{"index":6,"sense_key":"long%5:00:00:unsound:00","synset_id":"long.s.02"},{"index":7,"sense_key":"long%3:00:00::","synset_id":"retentive.a.01"},{"index":8,"sense_key":"long%3:00:04::","synset_id":"long.a.06"},{"index":9,"sense_key":"long%5:00:00:abundant:00","synset_id":"long.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"long%3:00:02::","call_ids":["senseval3.d002.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"long%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.005195,"input_uncached_usd":0.0034850000000000003,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.672767708078027}
{"item_id":"senseval3.d002.s017.t001","gold_sense_keys":["take%2:40:06::"],"candidates":[{"index":1,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":2,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":3,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":4,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":5,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":6,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":7,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":8,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":9,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":10,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":11,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":12,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":13,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":14,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":15,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":16,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":17,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":18,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":19,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":20,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":21,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":22,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":23,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":24,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":25,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":26,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":27,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":28,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":29,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":30,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":31,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":32,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":33,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":34,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":35,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":36,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":37,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":38,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":39,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":40,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":41,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"},{"index":42,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":34,"chosen_sense_key":"take%2:40:06::","call_ids":["senseval3.d002.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":34,"predicted_sense_key":"take%2:40:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2146,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.012890000000000002,"input_uncached_usd":0.010730000000000002,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9131805419456214}
{"item_id":"senseval3.d002.s017.t003","gold_sense_keys":["pass%2:30:01::","pass%2:38:03::"],"candidates":[{"index":1,"sense_key":"pass%2:42:08::","synset_id":"run.v.03"},{"index":2,"sense_key":"pass%2:38:02::","synset_id":"sink.v.03"},{"index":3,"sense_key":"pass%2:42:00::","synset_id":"spend.v.01"},{"index":4,"sense_key":"pass%2:38:01::","synset_id":"pass.v.07"},{"index":5,"sense_key":"pass%2:35:00::","synset_id":"guide.v.05"},{"index":6,"sense_key":"pass%2:32:00::","synset_id":"authorize.v.01"},{"index":7,"sense_key":"pass%2:29:00::","synset_id":"excrete.v.01"},{"index":8,"sense_key":"pass%2:30:01::","synset_id":"evanesce.v.01"},{"index":9,"sense_key":"pass%2:40:01::","synset_id":"pass.v.20"},{"index":10,"sense_key":"pass%2:30:00::","synset_id":"happen.v.01"},{"index":11,"sense_key":"pass%2:38:05::","synset_id":"travel_by.v.01"},{"index":12,"sense_key":"pass%2:38:03::","synset_id":"elapse.v.01"},{"index":13,"sense_key":"pass%2:38:00::","synset_id":"pass.v.01"},{"index":14,"sense_key":"pass%2:30:11::","synset_id":"die.v.01"},{"index":15,"sense_key":"pass%2:41:02::","synset_id":"legislate.v.01"},{"index":16,"sense_key":"pass%2:32:01::","synset_id":"communicate.v.01"},{"index":17,"sense_key":"pass%2:40:13::","synset_id":"fall.v.21"},{"index":18,"sense_key":"pass%2:41:07::","synset_id":"pass.v.16"},{"index":19,"sense_key":"pass%2:41:04::","synset_id":"pass.v.09"},{"index":20,"sense_key":"pass%2:41:11::","synset_id":"pass.v.17"},{"index":21,"sense_key":"pass%2:40:00::","synset_id":"pass.v.05"},{"index":22,"sense_key":"pass%2:40:02::","synset_id":"pass.v.18"},{"index":23,"sense_key":"pass%2:38:04::","synset_id":"pass.v.22"},{"index":24,"sense_key":"pass%2:41:08::","synset_id":"pass.v.14"},{"index":25,"sense_key":"pass%2:42:04::","synset_id":"exceed.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"pass%2:30:01::","call_ids":["senseval3.d002.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"pass%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1421,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":73},"cost":{"currency":"USD","total_usd":0.009745,"input_uncached_usd":0.007105,"input_cached_usd":0.0,"output_usd":0.00264,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4256293749203905}
{"item_id":"senseval3.d002.s018.t000","gold_sense_keys":["time%1:11:00::"],"candidates":[{"index":1,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":2,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"},{"index":3,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":6,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":7,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":8,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"time%1:11:00::","call_ids":["senseval3.d002.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"time%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":664,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":80},"cost":{"currency":"USD","total_usd":0.00617,"input_uncached_usd":0.0033200000000000005,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.220245291944593}
{"item_id":"senseval3.d002.s018.t001","gold_sense_keys":["get%2:30:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":2,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":3,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":4,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":5,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":6,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":7,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":8,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":9,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":10,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":11,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":12,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":13,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"},{"index":14,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":15,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":16,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":17,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":18,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":19,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":20,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":21,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":22,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":23,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":24,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":25,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":26,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":29,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":30,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":31,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":32,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":33,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":34,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":35,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":36,"sense_key":"get%2:40:00::","synset_id":"get.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"get%2:30:00::","call_ids":["senseval3.d002.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"get%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1893,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.011895000000000001,"input_uncached_usd":0.009465000000000001,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7427213750779629}
{"item_id":"senseval3.d002.s018.t002","gold_sense_keys":["building%1:06:00::"],"candidates":[{"index":1,"sense_key":"building%1:06:00::","synset_id":"building.n.01"},{"index":2,"sense_key":"building%1:04:00::","synset_id":"construction.n.01"},{"index":3,"sense_key":"building%1:04:01::","synset_id":"construction.n.07"},{"index":4,"sense_key":"building%1:14:00::","synset_id":"building.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"building%1:06:00::","call_ids":["senseval3.d002.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"building%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.00381,"input_uncached_usd":0.00222,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5943134580738842}
{"item_id":"senseval3.d002.s018.t003","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":2,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":5,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":6,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":7,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":8,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":10,"sense_key":"start%2:38:03::","synset_id":"start.v.14"},{"index":11,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":12,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":13,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":14,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"start%2:30:00::","call_ids":["senseval3.d002.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"start%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":887,"cached_input_tokens":0,"output_tokens":2048,"reasoning_output_tokens":2016},"cost":{"currency":"USD","total_usd":0.065875,"input_uncached_usd":0.004435000000000001,"input_cached_usd":0.0,"output_usd":0.06144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":35.40627166698687}
{"item_id":"senseval3.d002.s018.t004","gold_sense_keys":["shake%2:29:00::","shake%2:38:00::","shake%2:38:01::","shake%2:38:02::"],"candidates":[{"index":1,"sense_key":"shake%2:29:00::","synset_id":"shake.v.02"},{"index":2,"sense_key":"shake%2:30:01::","synset_id":"shake.v.08"},{"index":3,"sense_key":"shake%2:38:04::","synset_id":"shake.v.07"},{"index":4,"sense_key":"shake%2:38:01::","synset_id":"judder.v.01"},{"index":5,"sense_key":"shake%2:38:02::","synset_id":"rock.v.01"},{"index":6,"sense_key":"shake%2:30:00::","synset_id":"shake.v.05"},{"index":7,"sense_key":"shake%2:38:00::","synset_id":"shake.v.01"},{"index":8,"sense_key":"shake%2:32:00::","synset_id":"shake.v.09"},{"index":9,"sense_key":"shake%2:37:00::","synset_id":"stimulate.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"shake%2:38:01::","call_ids":["senseval3.d002.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"shake%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":2599,"reasoning_output_tokens":2562},"cost":{"currency":"USD","total_usd":0.081235,"input_uncached_usd":0.003265,"input_cached_usd":0.0,"output_usd":0.07797,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":40.13920791703276}
{"item_id":"senseval3.d002.s018.t005","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":2,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":3,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"child%1:18:00::","call_ids":["senseval3.d002.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"child%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.005115,"input_uncached_usd":0.0021750000000000003,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.376480167033151}
{"item_id":"senseval3.d002.s018.t006","gold_sense_keys":["toy%1:06:00::"],"candidates":[{"index":1,"sense_key":"toy%1:06:02::","synset_id":"toy.n.02"},{"index":2,"sense_key":"toy%1:05:00::","synset_id":"toy_dog.n.01"},{"index":3,"sense_key":"toy%1:06:01::","synset_id":"miniature.n.02"},{"index":4,"sense_key":"toy%1:06:03::","synset_id":"toy.n.03"},{"index":5,"sense_key":"toy%1:06:00::","synset_id":"plaything.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"toy%1:06:00::","call_ids":["senseval3.d002.s018.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"toy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":425,"cached_input_tokens":0,"output_tokens":199,"reasoning_output_tokens":183},"cost":{"currency":"USD","total_usd":0.008095000000000001,"input_uncached_usd":0.002125,"input_cached_usd":0.0,"output_usd":0.0059700000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8204862910788506}
{"item_id":"senseval3.d002.s018.t007","gold_sense_keys":["block%1:06:00::"],"candidates":[{"index":1,"sense_key":"block%1:15:00::","synset_id":"block.n.02"},{"index":2,"sense_key":"block%1:06:00::","synset_id":"block.n.01"},{"index":3,"sense_key":"block%1:09:00::","synset_id":"block.n.07"},{"index":4,"sense_key":"block%1:06:01::","synset_id":"blockage.n.02"},{"index":5,"sense_key":"block%1:06:06::","synset_id":"block.n.05"},{"index":6,"sense_key":"block%1:14:00::","synset_id":"block.n.04"},{"index":7,"sense_key":"block%1:06:04::","synset_id":"engine_block.n.01"},{"index":8,"sense_key":"block%1:23:00::","synset_id":"block.n.06"},{"index":9,"sense_key":"block%1:06:05::","synset_id":"auction_block.n.01"},{"index":10,"sense_key":"block%1:04:00::","synset_id":"blocking.n.01"},{"index":11,"sense_key":"block%1:25:01::","synset_id":"block.n.03"},{"index":12,"sense_key":"block%1:06:02::","synset_id":"pulley.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"block%1:06:00::","call_ids":["senseval3.d002.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"block%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":812,"cached_input_tokens":0,"output_tokens":440,"reasoning_output_tokens":424},"cost":{"currency":"USD","total_usd":0.01726,"input_uncached_usd":0.00406,"input_cached_usd":0.0,"output_usd":0.0132,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.199516291031614}
{"item_id":"senseval3.d002.s018.t008","gold_sense_keys":["toss%2:35:05::"],"candidates":[{"index":1,"sense_key":"toss%2:40:00::","synset_id":"discard.v.01"},{"index":2,"sense_key":"toss%2:35:00::","synset_id":"toss.v.06"},{"index":3,"sense_key":"toss%2:38:01::","synset_id":"convulse.v.03"},{"index":4,"sense_key":"toss%2:38:00::","synset_id":"flip.v.01"},{"index":5,"sense_key":"toss%2:35:04::","synset_id":"flip.v.06"},{"index":6,"sense_key":"toss%2:35:05::","synset_id":"chuck.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"toss%2:38:01::","call_ids":["senseval3.d002.s018.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"toss%2:38:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01839,"input_uncached_usd":0.00255,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.34486566693522}
{"item_id":"senseval3.d002.s021.t000","gold_sense_keys":["library%1:06:00::","library%1:06:01::","library%1:06:02::"],"candidates":[{"index":1,"sense_key":"library%1:14:00::","synset_id":"library.n.02"},{"index":2,"sense_key":"library%1:06:02::","synset_id":"library.n.05"},{"index":3,"sense_key":"library%1:06:01::","synset_id":"library.n.01"},{"index":4,"sense_key":"library%1:14:01::","synset_id":"library.n.04"},{"index":5,"sense_key":"library%1:06:00::","synset_id":"library.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"library%1:06:02::","call_ids":["senseval3.d002.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"library%1:06:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018009999999999998,"input_uncached_usd":0.00217,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.862995708012022}
{"item_id":"senseval3.d002.s021.t001","gold_sense_keys":["hit%2:35:05::"],"candidates":[{"index":1,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"},{"index":2,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":3,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":4,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"},{"index":5,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"},{"index":6,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":7,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":8,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":9,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":10,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":11,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":12,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"},{"index":13,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":14,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"},{"index":15,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":16,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"},{"index":17,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"hit%2:35:05::","call_ids":["senseval3.d002.s021.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"hit%2:35:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1002,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":110},"cost":{"currency":"USD","total_usd":0.008790000000000001,"input_uncached_usd":0.0050100000000000006,"input_cached_usd":0.0,"output_usd":0.00378,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7348614999791607}
{"item_id":"senseval3.d002.s022.t000","gold_sense_keys":["endless%5:00:00:long:02"],"candidates":[{"index":1,"sense_key":"endless%5:00:00:infinite:00","synset_id":"dateless.s.01"},{"index":2,"sense_key":"endless%5:00:00:long:02","synset_id":"endless.s.01"},{"index":3,"sense_key":"endless%5:00:00:continuous:01","synset_id":"endless.s.04"},{"index":4,"sense_key":"endless%5:00:02:infinite:00","synset_id":"endless.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"endless%5:00:00:long:02","call_ids":["senseval3.d002.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"endless%5:00:00:long:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.004135,"input_uncached_usd":0.0020350000000000004,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.873818333959207}
{"item_id":"senseval3.d002.s022.t001","gold_sense_keys":["second%1:28:00::"],"candidates":[{"index":1,"sense_key":"second%1:28:02::","synset_id":"moment.n.01"},{"index":2,"sense_key":"second%1:06:00::","synset_id":"second_gear.n.01"},{"index":3,"sense_key":"second%1:18:00::","synset_id":"second.n.07"},{"index":4,"sense_key":"second%1:10:00::","synset_id":"second.n.08"},{"index":5,"sense_key":"second%1:23:00::","synset_id":"second.n.06"},{"index":6,"sense_key":"second%1:28:01::","synset_id":"moment.n.02"},{"index":7,"sense_key":"second%1:04:00::","synset_id":"second_base.n.02"},{"index":8,"sense_key":"second%1:24:00::","synset_id":"second.n.05"},{"index":9,"sense_key":"second%1:28:00::","synset_id":"second.n.01"},{"index":10,"sense_key":"second%1:06:01::","synset_id":"irregular.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"second%1:28:00::","call_ids":["senseval3.d002.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"second%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":358,"reasoning_output_tokens":342},"cost":{"currency":"USD","total_usd":0.014005,"input_uncached_usd":0.003265,"input_cached_usd":0.0,"output_usd":0.01074,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.087773625040427}
{"item_id":"senseval3.d002.s022.t002","gold_sense_keys":["wonder%2:32:01::"],"candidates":[{"index":1,"sense_key":"wonder%2:31:00::","synset_id":"wonder.v.01"},{"index":2,"sense_key":"wonder%2:32:12::","synset_id":"wonder.v.03"},{"index":3,"sense_key":"wonder%2:32:01::","synset_id":"wonder.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"wonder%2:32:01::","call_ids":["senseval3.d002.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"wonder%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":366,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.00474,"input_uncached_usd":0.0018300000000000002,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4123427500016987}
{"item_id":"senseval3.d002.s022.t004","gold_sense_keys":["window%1:06:00::"],"candidates":[{"index":1,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":2,"sense_key":"window%1:28:00::","synset_id":"window.n.05"},{"index":3,"sense_key":"window%1:06:00::","synset_id":"window.n.01"},{"index":4,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":5,"sense_key":"window%1:06:01::","synset_id":"window.n.02"},{"index":6,"sense_key":"window%1:06:03::","synset_id":"window.n.08"},{"index":7,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":8,"sense_key":"window%1:17:00::","synset_id":"window.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"window%1:06:04::","call_ids":["senseval3.d002.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"window%1:06:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018675,"input_uncached_usd":0.0028350000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.979031750001013}
{"item_id":"senseval3.d002.s022.t005","gold_sense_keys":["buckle%2:30:00::","buckle%2:35:01::"],"candidates":[{"index":1,"sense_key":"buckle%2:35:01::","synset_id":"buckle.v.02"},{"index":2,"sense_key":"buckle%2:30:00::","synset_id":"heave.v.07"},{"index":3,"sense_key":"buckle%2:35:00::","synset_id":"buckle.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"buckle%2:30:00::","call_ids":["senseval3.d002.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"buckle%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":347,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":121},"cost":{"currency":"USD","total_usd":0.005845,"input_uncached_usd":0.0017350000000000002,"input_cached_usd":0.0,"output_usd":0.00411,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.52686033397913}
{"item_id":"senseval3.d002.s022.t006","gold_sense_keys":["shower%2:35:00::"],"candidates":[{"index":1,"sense_key":"shower%2:29:00::","synset_id":"shower.v.03"},{"index":2,"sense_key":"shower%2:40:00::","synset_id":"lavish.v.01"},{"index":3,"sense_key":"shower%2:35:00::","synset_id":"shower.v.02"},{"index":4,"sense_key":"shower%2:34:12::","synset_id":"shower.v.05"},{"index":5,"sense_key":"shower%2:43:00::","synset_id":"shower.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"shower%2:35:00::","call_ids":["senseval3.d002.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"shower%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":170,"reasoning_output_tokens":154},"cost":{"currency":"USD","total_usd":0.007210000000000001,"input_uncached_usd":0.0021100000000000003,"input_cached_usd":0.0,"output_usd":0.0051,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.748432749998756}
{"item_id":"senseval3.d002.s022.t007","gold_sense_keys":["glass%1:27:00::"],"candidates":[{"index":1,"sense_key":"glass%1:23:00::","synset_id":"glass.n.03"},{"index":2,"sense_key":"glass%1:06:02::","synset_id":"looking_glass.n.01"},{"index":3,"sense_key":"glass%1:27:00::","synset_id":"glass.n.01"},{"index":4,"sense_key":"glass%1:06:00::","synset_id":"glass.n.02"},{"index":5,"sense_key":"glass%1:06:01::","synset_id":"methamphetamine.n.01"},{"index":6,"sense_key":"glass%1:06:04::","synset_id":"glass.n.07"},{"index":7,"sense_key":"glass%1:06:03::","synset_id":"field_glass.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"glass%1:27:00::","call_ids":["senseval3.d002.s022.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"glass%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.004615,"input_uncached_usd":0.002395,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5018112499965355}
{"item_id":"senseval3.d002.s023.t001","gold_sense_keys":["book%1:06:00::"],"candidates":[{"index":1,"sense_key":"book%1:21:00::","synset_id":"ledger.n.01"},{"index":2,"sense_key":"book%1:10:05::","synset_id":"koran.n.01"},{"index":3,"sense_key":"book%1:14:00::","synset_id":"book.n.07"},{"index":4,"sense_key":"book%1:14:01::","synset_id":"book.n.06"},{"index":5,"sense_key":"book%1:06:00::","synset_id":"book.n.02"},{"index":6,"sense_key":"book%1:10:03::","synset_id":"book.n.10"},{"index":7,"sense_key":"book%1:10:04::","synset_id":"bible.n.01"},{"index":8,"sense_key":"book%1:10:00::","synset_id":"book.n.01"},{"index":9,"sense_key":"book%1:06:02::","synset_id":"book.n.11"},{"index":10,"sense_key":"book%1:10:01::","synset_id":"script.n.01"},{"index":11,"sense_key":"book%1:10:02::","synset_id":"record.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"book%1:06:00::","call_ids":["senseval3.d002.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"book%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.005810000000000001,"input_uncached_usd":0.0036200000000000004,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0999775420641527}
{"item_id":"senseval3.d002.s023.t002","gold_sense_keys":["fall%2:38:03::"],"candidates":[{"index":1,"sense_key":"fall%2:30:06::","synset_id":"decrease.v.01"},{"index":2,"sense_key":"fall%2:38:03::","synset_id":"fall.v.01"},{"index":3,"sense_key":"fall%2:30:14::","synset_id":"fall.v.27"},{"index":4,"sense_key":"fall%2:40:09::","synset_id":"accrue.v.02"},{"index":5,"sense_key":"fall%2:30:15::","synset_id":"fall.v.31"},{"index":6,"sense_key":"fall%2:40:12::","synset_id":"fall.v.18"},{"index":7,"sense_key":"fall%2:41:00::","synset_id":"fall.v.13"},{"index":8,"sense_key":"fall%2:40:03::","synset_id":"fall.v.14"},{"index":9,"sense_key":"fall%2:30:08::","synset_id":"fall.v.03"},{"index":10,"sense_key":"fall%2:42:03::","synset_id":"fall.v.04"},{"index":11,"sense_key":"fall%2:41:15::","synset_id":"fall.v.17"},{"index":12,"sense_key":"fall%2:41:02::","synset_id":"fall.v.06"},{"index":13,"sense_key":"fall%2:35:00::","synset_id":"hang.v.05"},{"index":14,"sense_key":"fall%2:38:01::","synset_id":"fall.v.15"},{"index":15,"sense_key":"fall%2:40:08::","synset_id":"fall.v.20"},{"index":16,"sense_key":"fall%2:40:05::","synset_id":"fall.v.09"},{"index":17,"sense_key":"fall%2:38:14::","synset_id":"fall.v.24"},{"index":18,"sense_key":"fall%2:42:04::","synset_id":"fall.v.08"},{"index":19,"sense_key":"fall%2:43:00::","synset_id":"precipitate.v.03"},{"index":20,"sense_key":"fall%2:42:00::","synset_id":"fall.v.07"},{"index":21,"sense_key":"fall%2:38:11::","synset_id":"fall.v.22"},{"index":22,"sense_key":"fall%2:30:09::","synset_id":"fall.v.29"},{"index":23,"sense_key":"fall%2:30:11::","synset_id":"fall.v.26"},{"index":24,"sense_key":"fall%2:41:01::","synset_id":"fall.v.12"},{"index":25,"sense_key":"fall%2:30:13::","synset_id":"fall.v.32"},{"index":26,"sense_key":"fall%2:30:10::","synset_id":"fall.v.28"},{"index":27,"sense_key":"fall%2:30:12::","synset_id":"fall.v.30"},{"index":28,"sense_key":"fall%2:30:00::","synset_id":"fall.v.10"},{"index":29,"sense_key":"fall%2:40:13::","synset_id":"fall.v.21"},{"index":30,"sense_key":"fall%2:38:15::","synset_id":"fall.v.23"},{"index":31,"sense_key":"fall%2:38:00::","synset_id":"descend.v.01"},{"index":32,"sense_key":"fall%2:42:02::","synset_id":"fall.v.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fall%2:38:03::","call_ids":["senseval3.d002.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fall%2:38:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1502,"cached_input_tokens":0,"output_tokens":285,"reasoning_output_tokens":269},"cost":{"currency":"USD","total_usd":0.01606,"input_uncached_usd":0.00751,"input_cached_usd":0.0,"output_usd":0.00855,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.847906540962867}
{"item_id":"senseval3.d002.s024.t001","gold_sense_keys":["paint%1:06:00::"],"candidates":[{"index":1,"sense_key":"paint%1:15:00::","synset_id":"key.n.09"},{"index":2,"sense_key":"paint%1:06:01::","synset_id":"rouge.n.01"},{"index":3,"sense_key":"paint%1:06:00::","synset_id":"paint.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"paint%1:06:00::","call_ids":["senseval3.d002.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"paint%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":398,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":95},"cost":{"currency":"USD","total_usd":0.0052899999999999996,"input_uncached_usd":0.00199,"input_cached_usd":0.0,"output_usd":0.0033,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.246951333945617}
{"item_id":"senseval3.d002.s024.t002","gold_sense_keys":["shop%1:06:01::"],"candidates":[{"index":1,"sense_key":"shop%1:06:00::","synset_id":"shop.n.01"},{"index":2,"sense_key":"shop%1:04:00::","synset_id":"shop_class.n.01"},{"index":3,"sense_key":"shop%1:06:01::","synset_id":"workshop.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"shop%1:06:01::","call_ids":["senseval3.d002.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"shop%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":340,"cached_input_tokens":0,"output_tokens":189,"reasoning_output_tokens":173},"cost":{"currency":"USD","total_usd":0.007370000000000001,"input_uncached_usd":0.0017000000000000001,"input_cached_usd":0.0,"output_usd":0.0056700000000000006,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.71839541697409}
{"item_id":"senseval3.d002.s024.t003","gold_sense_keys":["fire%1:11:00::"],"candidates":[{"index":1,"sense_key":"fire%1:10:00::","synset_id":"fire.n.09"},{"index":2,"sense_key":"fire%1:27:00::","synset_id":"fire.n.07"},{"index":3,"sense_key":"fire%1:12:00::","synset_id":"ardor.n.03"},{"index":4,"sense_key":"fire%1:27:01::","synset_id":"fire.n.05"},{"index":5,"sense_key":"fire%1:22:00::","synset_id":"fire.n.03"},{"index":6,"sense_key":"fire%1:11:01::","synset_id":"fire.n.08"},{"index":7,"sense_key":"fire%1:06:00::","synset_id":"fire.n.04"},{"index":8,"sense_key":"fire%1:04:00::","synset_id":"fire.n.02"},{"index":9,"sense_key":"fire%1:11:00::","synset_id":"fire.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"fire%1:11:00::","call_ids":["senseval3.d002.s024.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"fire%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.005755000000000001,"input_uncached_usd":0.0030550000000000004,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0764170408947393}
{"item_id":"senseval3.d002.s024.t004","gold_sense_keys":["send%2:38:01::"],"candidates":[{"index":1,"sense_key":"send%2:35:00::","synset_id":"send.v.02"},{"index":2,"sense_key":"send%2:38:00::","synset_id":"transport.v.04"},{"index":3,"sense_key":"send%2:33:00::","synset_id":"station.v.01"},{"index":4,"sense_key":"send%2:38:01::","synset_id":"send.v.01"},{"index":5,"sense_key":"send%2:32:03::","synset_id":"air.v.03"},{"index":6,"sense_key":"send%2:32:00::","synset_id":"mail.v.02"},{"index":7,"sense_key":"send%2:40:00::","synset_id":"commit.v.03"},{"index":8,"sense_key":"send%2:32:02::","synset_id":"send.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"send%2:38:01::","call_ids":["senseval3.d002.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"send%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":562,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.005180000000000001,"input_uncached_usd":0.0028100000000000004,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7059936659643427}
{"item_id":"senseval3.d002.s024.t006","gold_sense_keys":["cloud%1:19:01::"],"candidates":[{"index":1,"sense_key":"cloud%1:19:01::","synset_id":"cloud.n.01"},{"index":2,"sense_key":"cloud%1:17:00::","synset_id":"cloud.n.02"},{"index":3,"sense_key":"cloud%1:26:00::","synset_id":"cloud.n.05"},{"index":4,"sense_key":"cloud%1:26:02::","synset_id":"cloud.n.04"},{"index":5,"sense_key":"cloud%1:26:01::","synset_id":"cloud.n.03"},{"index":6,"sense_key":"cloud%1:14:00::","synset_id":"swarm.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cloud%1:19:01::","call_ids":["senseval3.d002.s024.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cloud%1:19:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.0038150000000000002,"input_uncached_usd":0.0021650000000000003,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.511856040917337}
{"item_id":"senseval3.d002.s024.t007","gold_sense_keys":["black%3:00:01::"],"candidates":[{"index":1,"sense_key":"black%5:00:00:evil:00","synset_id":"black.s.03"},{"index":2,"sense_key":"black%5:00:00:sarcastic:00","synset_id":"black.s.07"},{"index":3,"sense_key":"black%5:00:00:hopeless:00","synset_id":"black.s.02"},{"index":4,"sense_key":"black%3:00:01::","synset_id":"black.a.01"},{"index":5,"sense_key":"black%5:00:00:dirty:01","synset_id":"black.s.12"},{"index":6,"sense_key":"black%5:00:00:dark:01","synset_id":"black.s.06"},{"index":7,"sense_key":"black%5:00:00:angry:00","synset_id":"black.s.01"},{"index":8,"sense_key":"black%5:00:00:covert:00","synset_id":"black.s.08"},{"index":9,"sense_key":"black%3:00:02::","synset_id":"black.a.02"},{"index":10,"sense_key":"black%5:00:00:unfortunate:00","synset_id":"black.s.04"},{"index":11,"sense_key":"black%5:00:00:colored:00","synset_id":"black.s.05"},{"index":12,"sense_key":"black%5:00:00:dishonorable:00","synset_id":"black.s.10"},{"index":13,"sense_key":"black%5:00:00:undiluted:00","synset_id":"black.s.11"},{"index":14,"sense_key":"black%5:00:00:illegal:00","synset_id":"bootleg.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"black%3:00:01::","call_ids":["senseval3.d002.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"black%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":932,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.00781,"input_uncached_usd":0.00466,"input_cached_usd":0.0,"output_usd":0.00315,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2775452909991145}
{"item_id":"senseval3.d002.s024.t008","gold_sense_keys":["smoke%1:19:00::","smoke%1:22:00::"],"candidates":[{"index":1,"sense_key":"smoke%1:19:00::","synset_id":"smoke.n.01"},{"index":2,"sense_key":"smoke%1:06:00::","synset_id":"roll_of_tobacco.n.01"},{"index":3,"sense_key":"smoke%1:22:00::","synset_id":"smoke.n.02"},{"index":4,"sense_key":"smoke%1:06:01::","synset_id":"pot.n.09"},{"index":5,"sense_key":"smoke%1:04:00::","synset_id":"smoke.n.07"},{"index":6,"sense_key":"smoke%1:07:00::","synset_id":"smoke.n.04"},{"index":7,"sense_key":"smoke%1:10:00::","synset_id":"smoke.n.03"},{"index":8,"sense_key":"smoke%1:04:02::","synset_id":"fastball.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"smoke%1:22:00::","call_ids":["senseval3.d002.s024.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"smoke%1:22:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":550,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":85},"cost":{"currency":"USD","total_usd":0.00578,"input_uncached_usd":0.0027500000000000003,"input_cached_usd":0.0,"output_usd":0.00303,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.866347459028475}
{"item_id":"senseval3.d002.s024.t009","gold_sense_keys":["air%1:15:00::","air%1:27:00::"],"candidates":[{"index":1,"sense_key":"air%1:19:00::","synset_id":"breeze.n.01"},{"index":2,"sense_key":"air%1:27:01::","synset_id":"air.n.06"},{"index":3,"sense_key":"air%1:27:00::","synset_id":"air.n.01"},{"index":4,"sense_key":"air%1:10:02::","synset_id":"air.n.08"},{"index":5,"sense_key":"air%1:10:01::","synset_id":"tune.n.01"},{"index":6,"sense_key":"air%1:15:00::","synset_id":"air.n.02"},{"index":7,"sense_key":"air%1:07:00::","synset_id":"air.n.03"},{"index":8,"sense_key":"air%1:04:00::","synset_id":"air_travel.n.01"},{"index":9,"sense_key":"air%1:15:01::","synset_id":"atmosphere.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"air%1:15:00::","call_ids":["senseval3.d002.s024.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"air%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":641,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019045,"input_uncached_usd":0.0032050000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.86345429206267}
{"item_id":"senseval3.d002.s027.t001","gold_sense_keys":["fine%5:00:00:satisfactory:00"],"candidates":[{"index":1,"sense_key":"fine%3:00:00::","synset_id":"fine.a.05"},{"index":2,"sense_key":"fine%5:00:00:pure:02","synset_id":"fine.s.05"},{"index":3,"sense_key":"fine%5:00:00:precise:00","synset_id":"fine.s.02"},{"index":4,"sense_key":"fine%5:00:00:thin:01","synset_id":"fine.s.03"},{"index":5,"sense_key":"fine%5:00:00:elegant:00","synset_id":"fine.s.04"},{"index":6,"sense_key":"fine%5:00:00:satisfactory:00","synset_id":"all_right.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"fine%5:00:00:satisfactory:00","call_ids":["senseval3.d002.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"fine%5:00:00:satisfactory:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00426,"input_uncached_usd":0.00237,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7985135409981012}
{"item_id":"senseval3.d002.s028.t000","gold_sense_keys":["building%1:06:00::"],"candidates":[{"index":1,"sense_key":"building%1:04:01::","synset_id":"construction.n.07"},{"index":2,"sense_key":"building%1:04:00::","synset_id":"construction.n.01"},{"index":3,"sense_key":"building%1:06:00::","synset_id":"building.n.01"},{"index":4,"sense_key":"building%1:14:00::","synset_id":"building.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"building%1:06:00::","call_ids":["senseval3.d002.s028.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"building%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.003455,"input_uncached_usd":0.002075,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8359015830792487}
{"item_id":"senseval3.d002.s028.t001","gold_sense_keys":["shake%2:29:00::","shake%2:38:00::","shake%2:38:01::","shake%2:38:02::"],"candidates":[{"index":1,"sense_key":"shake%2:30:00::","synset_id":"shake.v.05"},{"index":2,"sense_key":"shake%2:38:01::","synset_id":"judder.v.01"},{"index":3,"sense_key":"shake%2:32:00::","synset_id":"shake.v.09"},{"index":4,"sense_key":"shake%2:37:00::","synset_id":"stimulate.v.03"},{"index":5,"sense_key":"shake%2:29:00::","synset_id":"shake.v.02"},{"index":6,"sense_key":"shake%2:38:00::","synset_id":"shake.v.01"},{"index":7,"sense_key":"shake%2:30:01::","synset_id":"shake.v.08"},{"index":8,"sense_key":"shake%2:38:02::","synset_id":"rock.v.01"},{"index":9,"sense_key":"shake%2:38:04::","synset_id":"shake.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"shake%2:38:01::","call_ids":["senseval3.d002.s028.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"shake%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01896,"input_uncached_usd":0.0031200000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.564878541976213}
{"item_id":"senseval3.d002.s028.t002","gold_sense_keys":["like_hell%4:02:00::"],"candidates":[{"index":1,"sense_key":"like_hell%4:02:01::","synset_id":"like_hell.r.02"},{"index":2,"sense_key":"like_hell%4:02:00::","synset_id":"like_hell.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"like_hell%4:02:00::","call_ids":["senseval3.d002.s028.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"like_hell%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":334,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.0035,"input_uncached_usd":0.00167,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3449295829050243}
{"item_id":"senseval3.d002.s028.t003","gold_sense_keys":["keep%2:42:07::"],"candidates":[{"index":1,"sense_key":"keep%2:41:03::","synset_id":"observe.v.09"},{"index":2,"sense_key":"keep%2:40:13::","synset_id":"keep.v.18"},{"index":3,"sense_key":"keep%2:42:03::","synset_id":"keep.v.12"},{"index":4,"sense_key":"keep%2:41:05::","synset_id":"observe.v.06"},{"index":5,"sense_key":"keep%2:42:07::","synset_id":"continue.v.01"},{"index":6,"sense_key":"keep%2:40:12::","synset_id":"keep.v.19"},{"index":7,"sense_key":"keep%2:42:02::","synset_id":"keep.v.15"},{"index":8,"sense_key":"keep%2:42:12::","synset_id":"keep.v.16"},{"index":9,"sense_key":"keep%2:35:10::","synset_id":"keep.v.21"},{"index":10,"sense_key":"keep%2:40:01::","synset_id":"keep.v.07"},{"index":11,"sense_key":"keep%2:42:01::","synset_id":"keep.v.09"},{"index":12,"sense_key":"keep%2:41:01::","synset_id":"prevent.v.02"},{"index":13,"sense_key":"keep%2:34:00::","synset_id":"sustain.v.04"},{"index":14,"sense_key":"keep%2:40:10::","synset_id":"keep.v.20"},{"index":15,"sense_key":"keep%2:42:00::","synset_id":"keep.v.01"},{"index":16,"sense_key":"keep%2:41:02::","synset_id":"retain.v.02"},{"index":17,"sense_key":"keep%2:40:09::","synset_id":"keep_open.v.01"},{"index":18,"sense_key":"keep%2:41:00::","synset_id":"restrain.v.01"},{"index":19,"sense_key":"keep%2:31:00::","synset_id":"observe.v.08"},{"index":20,"sense_key":"keep%2:40:00::","synset_id":"keep.v.03"},{"index":21,"sense_key":"keep%2:30:10::","synset_id":"preserve.v.04"},{"index":22,"sense_key":"keep%2:32:00::","synset_id":"keep.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"keep%2:42:07::","call_ids":["senseval3.d002.s028.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"keep%2:42:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1163,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":121},"cost":{"currency":"USD","total_usd":0.009895000000000001,"input_uncached_usd":0.005815000000000001,"input_cached_usd":0.0,"output_usd":0.00408,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8477337500080466}
{"item_id":"senseval3.d002.s028.t004","gold_sense_keys":["get%2:30:00::"],"candidates":[{"index":1,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":2,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":3,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":4,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":5,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":6,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":7,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":8,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":9,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":10,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":11,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":14,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":15,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":16,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":17,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":20,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":24,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":25,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":26,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":27,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":28,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":29,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":30,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":31,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":32,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":33,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"},{"index":34,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":35,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":36,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"get%2:30:00::","call_ids":["senseval3.d002.s028.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"get%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1864,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":135},"cost":{"currency":"USD","total_usd":0.013850000000000001,"input_uncached_usd":0.00932,"input_cached_usd":0.0,"output_usd":0.00453,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7959861250128597}
{"item_id":"senseval3.d002.s029.t001","gold_sense_keys":["explode%2:30:04::"],"candidates":[{"index":1,"sense_key":"explode%2:31:00::","synset_id":"explode.v.03"},{"index":2,"sense_key":"explode%2:32:00::","synset_id":"explode.v.07"},{"index":3,"sense_key":"explode%2:31:01::","synset_id":"explode.v.08"},{"index":4,"sense_key":"explode%2:30:04::","synset_id":"detonate.v.02"},{"index":5,"sense_key":"explode%2:30:03::","synset_id":"explode.v.04"},{"index":6,"sense_key":"explode%2:35:00::","synset_id":"explode.v.05"},{"index":7,"sense_key":"explode%2:30:02::","synset_id":"explode.v.10"},{"index":8,"sense_key":"explode%2:30:00::","synset_id":"explode.v.02"},{"index":9,"sense_key":"explode%2:30:01::","synset_id":"explode.v.01"},{"index":10,"sense_key":"explode%2:32:02::","synset_id":"explode.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"explode%2:30:04::","call_ids":["senseval3.d002.s029.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"explode%2:30:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":133},"cost":{"currency":"USD","total_usd":0.007535000000000002,"input_uncached_usd":0.0030650000000000004,"input_cached_usd":0.0,"output_usd":0.00447,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.085867916001007}
{"item_id":"senseval3.d002.s029.t002","gold_sense_keys":["burn%2:30:00::"],"candidates":[{"index":1,"sense_key":"burn%2:43:02::","synset_id":"burn.v.05"},{"index":2,"sense_key":"burn%2:37:00::","synset_id":"burn.v.06"},{"index":3,"sense_key":"burn%2:36:05::","synset_id":"cut.v.21"},{"index":4,"sense_key":"burn%2:30:08::","synset_id":"cauterize.v.01"},{"index":5,"sense_key":"burn%2:30:00::","synset_id":"burn.v.03"},{"index":6,"sense_key":"burn%2:40:01::","synset_id":"burn.v.09"},{"index":7,"sense_key":"burn%2:34:00::","synset_id":"burn_off.v.01"},{"index":8,"sense_key":"burn%2:29:00::","synset_id":"sunburn.v.01"},{"index":9,"sense_key":"burn%2:39:04::","synset_id":"burn.v.10"},{"index":10,"sense_key":"burn%2:30:03::","synset_id":"burn.v.15"},{"index":11,"sense_key":"burn%2:39:00::","synset_id":"bite.v.02"},{"index":12,"sense_key":"burn%2:43:01::","synset_id":"burn.v.02"},{"index":13,"sense_key":"burn%2:30:01::","synset_id":"burn.v.07"},{"index":14,"sense_key":"burn%2:30:02::","synset_id":"burn.v.01"},{"index":15,"sense_key":"burn%2:41:00::","synset_id":"burn.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"burn%2:30:00::","call_ids":["senseval3.d002.s029.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"burn%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.006475,"input_uncached_usd":0.004015,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5831845419015735}
{"item_id":"senseval3.d002.s029.t004","gold_sense_keys":["thing%1:09:02::"],"candidates":[{"index":1,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":2,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":3,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":4,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":7,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"},{"index":8,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":9,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":10,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":11,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":12,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"thing%1:26:00::","call_ids":["senseval3.d002.s029.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"thing%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":209,"reasoning_output_tokens":194},"cost":{"currency":"USD","total_usd":0.0094,"input_uncached_usd":0.0031300000000000004,"input_cached_usd":0.0,"output_usd":0.00627,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5097384579712525}
{"item_id":"senseval3.d002.s029.t005","gold_sense_keys":["peaceful%3:00:00::"],"candidates":[{"index":1,"sense_key":"peaceful%3:00:00::","synset_id":"peaceful.a.01"},{"index":2,"sense_key":"peaceful%5:00:00:nonviolent:00","synset_id":"passive.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"peaceful%3:00:00::","call_ids":["senseval3.d002.s029.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"peaceful%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":309,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.0038250000000000003,"input_uncached_usd":0.0015450000000000001,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5805582909379154}
{"item_id":"senseval3.d002.s030.t000","gold_sense_keys":["lot%1:23:00::"],"candidates":[{"index":1,"sense_key":"lot%1:15:00::","synset_id":"lot.n.02"},{"index":2,"sense_key":"lot%1:06:00::","synset_id":"draw.n.04"},{"index":3,"sense_key":"lot%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"lot%1:26:00::","synset_id":"fortune.n.04"},{"index":5,"sense_key":"lot%1:14:00::","synset_id":"set.n.05"},{"index":6,"sense_key":"lot%1:14:01::","synset_id":"bunch.n.03"},{"index":7,"sense_key":"lot%1:18:00::","synset_id":"lot.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"lot%1:23:00::","call_ids":["senseval3.d002.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"lot%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00459,"input_uncached_usd":0.0027600000000000003,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6851746250176802}
{"item_id":"senseval3.d002.s030.t001","gold_sense_keys":["car%1:06:00::"],"candidates":[{"index":1,"sense_key":"car%1:06:00::","synset_id":"car.n.01"},{"index":2,"sense_key":"car%1:06:01::","synset_id":"car.n.02"},{"index":3,"sense_key":"car%1:06:02::","synset_id":"car.n.04"},{"index":4,"sense_key":"car%1:06:03::","synset_id":"car.n.03"},{"index":5,"sense_key":"car%1:06:04::","synset_id":"cable_car.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"car%1:06:00::","call_ids":["senseval3.d002.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"car%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00398,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0407029159832746}
{"item_id":"senseval3.d002.s030.t002","gold_sense_keys":["alarm%1:06:00::"],"candidates":[{"index":1,"sense_key":"alarm%1:06:00::","synset_id":"alarm.n.02"},{"index":2,"sense_key":"alarm%1:10:00::","synset_id":"alarm.n.03"},{"index":3,"sense_key":"alarm%1:12:00::","synset_id":"alarm.n.01"},{"index":4,"sense_key":"alarm%1:06:01::","synset_id":"alarm_clock.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"alarm%1:06:00::","call_ids":["senseval3.d002.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"alarm%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":355,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017615,"input_uncached_usd":0.001775,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.400557541986927}
{"item_id":"senseval3.d002.s030.t003","gold_sense_keys":["go_off%2:30:00::"],"candidates":[{"index":1,"sense_key":"go_off%2:30:01::","synset_id":"go_off.v.05"},{"index":2,"sense_key":"go_off%2:30:00::","synset_id":"go_off.v.02"},{"index":3,"sense_key":"go_off%2:35:00::","synset_id":"go_off.v.04"},{"index":4,"sense_key":"go_off%2:33:00::","synset_id":"fire.v.05"},{"index":5,"sense_key":"go_off%2:30:02::","synset_id":"implode.v.01"},{"index":6,"sense_key":"go_off%2:38:00::","synset_id":"abscond.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"go_off%2:30:00::","call_ids":["senseval3.d002.s030.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"go_off%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.005295,"input_uncached_usd":0.002295,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9359334170585498}
{"item_id":"senseval3.d002.s031.t000","gold_sense_keys":["cat%1:05:00::"],"candidates":[{"index":1,"sense_key":"cat%1:18:01::","synset_id":"guy.n.01"},{"index":2,"sense_key":"cat%1:05:00::","synset_id":"cat.n.01"},{"index":3,"sense_key":"cat%1:06:01::","synset_id":"caterpillar.n.02"},{"index":4,"sense_key":"cat%1:05:02::","synset_id":"big_cat.n.01"},{"index":5,"sense_key":"cat%1:06:00::","synset_id":"cat-o'-nine-tails.n.01"},{"index":6,"sense_key":"cat%1:06:02::","synset_id":"kat.n.01"},{"index":7,"sense_key":"cat%1:18:00::","synset_id":"cat.n.03"},{"index":8,"sense_key":"cat%1:04:00::","synset_id":"computerized_tomography.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cat%1:05:00::","call_ids":["senseval3.d002.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cat%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.00476,"input_uncached_usd":0.00299,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2327814999734983}
{"item_id":"senseval3.d002.s031.t001","gold_sense_keys":["fine%5:00:00:satisfactory:00"],"candidates":[{"index":1,"sense_key":"fine%5:00:00:elegant:00","synset_id":"fine.s.04"},{"index":2,"sense_key":"fine%5:00:00:precise:00","synset_id":"fine.s.02"},{"index":3,"sense_key":"fine%5:00:00:thin:01","synset_id":"fine.s.03"},{"index":4,"sense_key":"fine%5:00:00:pure:02","synset_id":"fine.s.05"},{"index":5,"sense_key":"fine%5:00:00:satisfactory:00","synset_id":"all_right.s.01"},{"index":6,"sense_key":"fine%3:00:00::","synset_id":"fine.a.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"fine%5:00:00:satisfactory:00","call_ids":["senseval3.d002.s031.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"fine%5:00:00:satisfactory:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":40,"reasoning_output_tokens":25},"cost":{"currency":"USD","total_usd":0.0035550000000000004,"input_uncached_usd":0.0023550000000000003,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.492598499986343}
{"item_id":"senseval3.d002.s031.t002","gold_sense_keys":["nervous%5:00:00:troubled:00"],"candidates":[{"index":1,"sense_key":"nervous%5:00:00:troubled:00","synset_id":"anxious.s.02"},{"index":2,"sense_key":"nervous%5:00:00:excited:00","synset_id":"aflutter.s.01"},{"index":3,"sense_key":"nervous%5:00:00:excitable:00","synset_id":"skittish.s.01"},{"index":4,"sense_key":"nervous%3:01:00::","synset_id":"nervous.a.03"},{"index":5,"sense_key":"nervous%5:00:00:tense:03","synset_id":"nervous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nervous%5:00:00:troubled:00","call_ids":["senseval3.d002.s031.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nervous%5:00:00:troubled:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":234,"reasoning_output_tokens":218},"cost":{"currency":"USD","total_usd":0.00919,"input_uncached_usd":0.00217,"input_cached_usd":0.0,"output_usd":0.00702,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.88222541694995}
{"item_id":"senseval3.d002.s034.t001","gold_sense_keys":["fire%1:11:00::"],"candidates":[{"index":1,"sense_key":"fire%1:11:00::","synset_id":"fire.n.01"},{"index":2,"sense_key":"fire%1:12:00::","synset_id":"ardor.n.03"},{"index":3,"sense_key":"fire%1:04:00::","synset_id":"fire.n.02"},{"index":4,"sense_key":"fire%1:22:00::","synset_id":"fire.n.03"},{"index":5,"sense_key":"fire%1:11:01::","synset_id":"fire.n.08"},{"index":6,"sense_key":"fire%1:10:00::","synset_id":"fire.n.09"},{"index":7,"sense_key":"fire%1:27:00::","synset_id":"fire.n.07"},{"index":8,"sense_key":"fire%1:27:01::","synset_id":"fire.n.05"},{"index":9,"sense_key":"fire%1:06:00::","synset_id":"fire.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fire%1:11:00::","call_ids":["senseval3.d002.s034.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fire%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":129},"cost":{"currency":"USD","total_usd":0.007430000000000001,"input_uncached_usd":0.0030800000000000003,"input_cached_usd":0.0,"output_usd":0.00435,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.734483917010948}
{"item_id":"senseval3.d002.s035.t001","gold_sense_keys":["fill%1:27:00::"],"candidates":[{"index":1,"sense_key":"fill%1:27:00::","synset_id":"filling.n.01"},{"index":2,"sense_key":"fill%1:23:00::","synset_id":"fill.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fill%1:27:00::","call_ids":["senseval3.d002.s035.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fill%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":299,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.004195,"input_uncached_usd":0.001495,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3140615829033777}
{"item_id":"senseval3.d002.s035.t002","gold_sense_keys":["liquefy%2:30:00::","liquefy%2:43:00::"],"candidates":[{"index":1,"sense_key":"liquefy%2:43:00::","synset_id":"liquefy.v.01"},{"index":2,"sense_key":"liquefy%2:30:00::","synset_id":"liquefy.v.03"},{"index":3,"sense_key":"liquefy%2:30:01::","synset_id":"liquefy.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"liquefy%2:43:00::","call_ids":["senseval3.d002.s035.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"liquefy%2:43:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":359,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":88},"cost":{"currency":"USD","total_usd":0.0048850000000000005,"input_uncached_usd":0.0017950000000000002,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.811684042098932}
{"item_id":"senseval3.d002.s036.t000","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":2,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval3.d002.s036.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":423,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.0038850000000000004,"input_uncached_usd":0.002115,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1429154590005055}
{"item_id":"senseval3.d002.s036.t003","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%3:00:00::","synset_id":"able.a.01"},{"index":2,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"},{"index":3,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"},{"index":4,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"able%3:00:00::","call_ids":["senseval3.d002.s036.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"able%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":412,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":109},"cost":{"currency":"USD","total_usd":0.00581,"input_uncached_usd":0.00206,"input_cached_usd":0.0,"output_usd":0.0037500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.526901667006314}
{"item_id":"senseval3.d002.s036.t004","gold_sense_keys":["walk%2:38:00::"],"candidates":[{"index":1,"sense_key":"walk%2:41:01::","synset_id":"walk.v.07"},{"index":2,"sense_key":"walk%2:38:03::","synset_id":"walk.v.04"},{"index":3,"sense_key":"walk%2:33:01::","synset_id":"walk.v.05"},{"index":4,"sense_key":"walk%2:38:01::","synset_id":"walk.v.02"},{"index":5,"sense_key":"walk%2:41:00::","synset_id":"walk.v.06"},{"index":6,"sense_key":"walk%2:38:05::","synset_id":"walk.v.09"},{"index":7,"sense_key":"walk%2:33:00::","synset_id":"walk.v.03"},{"index":8,"sense_key":"walk%2:38:02::","synset_id":"walk.v.08"},{"index":9,"sense_key":"walk%2:38:00::","synset_id":"walk.v.01"},{"index":10,"sense_key":"walk%2:38:04::","synset_id":"walk.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"walk%2:38:00::","call_ids":["senseval3.d002.s036.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"walk%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":566,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":133},"cost":{"currency":"USD","total_usd":0.007300000000000001,"input_uncached_usd":0.00283,"input_cached_usd":0.0,"output_usd":0.00447,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.137576499953866}
{"item_id":"senseval3.d002.s036.t005","gold_sense_keys":["window%1:06:00::"],"candidates":[{"index":1,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":2,"sense_key":"window%1:28:00::","synset_id":"window.n.05"},{"index":3,"sense_key":"window%1:06:03::","synset_id":"window.n.08"},{"index":4,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":5,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":6,"sense_key":"window%1:06:01::","synset_id":"window.n.02"},{"index":7,"sense_key":"window%1:17:00::","synset_id":"window.n.04"},{"index":8,"sense_key":"window%1:06:00::","synset_id":"window.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"window%1:06:00::","call_ids":["senseval3.d002.s036.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"window%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":554,"cached_input_tokens":0,"output_tokens":248,"reasoning_output_tokens":232},"cost":{"currency":"USD","total_usd":0.01021,"input_uncached_usd":0.0027700000000000003,"input_cached_usd":0.0,"output_usd":0.00744,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.027384250075556}
{"item_id":"senseval3.d002.s036.t007","gold_sense_keys":["floor%1:06:01::"],"candidates":[{"index":1,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"},{"index":2,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":3,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"},{"index":4,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"},{"index":5,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"},{"index":6,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":7,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":8,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":9,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"},{"index":10,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"floor%1:06:01::","call_ids":["senseval3.d002.s036.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"floor%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.005215,"input_uncached_usd":0.003115,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8543459579814225}
{"item_id":"senseval3.d002.s036.t008","gold_sense_keys":["street%1:06:00::"],"candidates":[{"index":1,"sense_key":"street%1:26:01::","synset_id":"street.n.04"},{"index":2,"sense_key":"street%1:06:00::","synset_id":"street.n.01"},{"index":3,"sense_key":"street%1:26:00::","synset_id":"street.n.03"},{"index":4,"sense_key":"street%1:14:00::","synset_id":"street.n.05"},{"index":5,"sense_key":"street%1:06:01::","synset_id":"street.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"street%1:06:00::","call_ids":["senseval3.d002.s036.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"street%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":167,"reasoning_output_tokens":151},"cost":{"currency":"USD","total_usd":0.007220000000000001,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.00501,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.392934249946848}
{"item_id":"senseval3.d002.s036.t009","gold_sense_keys":["level%1:07:01::"],"candidates":[{"index":1,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"},{"index":2,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":3,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":4,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":5,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":6,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":7,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":8,"sense_key":"level%1:06:00::","synset_id":"level.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"level%1:07:01::","call_ids":["senseval3.d002.s036.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"level%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":295,"reasoning_output_tokens":279},"cost":{"currency":"USD","total_usd":0.011785,"input_uncached_usd":0.002935,"input_cached_usd":0.0,"output_usd":0.00885,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.882627625018358}
{"item_id":"senseval3.d002.s037.t000","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":2,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":3,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":4,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":5,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":6,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":7,"sense_key":"house%1:06:01::","synset_id":"house.n.12"},{"index":8,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":9,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":10,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":11,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":12,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s037.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.005525,"input_uncached_usd":0.0034850000000000003,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3374525420367718}
{"item_id":"senseval3.d002.s037.t001","gold_sense_keys":["ground%1:17:00::"],"candidates":[{"index":1,"sense_key":"ground%1:17:00::","synset_id":"land.n.04"},{"index":2,"sense_key":"ground%1:15:00::","synset_id":"ground.n.05"},{"index":3,"sense_key":"ground%1:06:00::","synset_id":"flat_coat.n.01"},{"index":4,"sense_key":"ground%1:27:00::","synset_id":"earth.n.02"},{"index":5,"sense_key":"ground%1:09:00::","synset_id":"background.n.02"},{"index":6,"sense_key":"ground%1:24:00::","synset_id":"footing.n.02"},{"index":7,"sense_key":"ground%1:06:01::","synset_id":"ground.n.10"},{"index":8,"sense_key":"ground%1:09:01::","synset_id":"ground.n.08"},{"index":9,"sense_key":"ground%1:06:02::","synset_id":"ground.n.09"},{"index":10,"sense_key":"ground%1:16:00::","synset_id":"reason.n.01"},{"index":11,"sense_key":"ground%1:17:01::","synset_id":"land.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"ground%1:27:00::","call_ids":["senseval3.d002.s037.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"ground%1:27:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":305,"reasoning_output_tokens":289},"cost":{"currency":"USD","total_usd":0.012865000000000001,"input_uncached_usd":0.0037150000000000004,"input_cached_usd":0.0,"output_usd":0.00915,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.2693175410386175}
{"item_id":"senseval3.d002.s040.t000","gold_sense_keys":["drive%2:38:01::"],"candidates":[{"index":1,"sense_key":"drive%2:35:01::","synset_id":"drive.v.09"},{"index":2,"sense_key":"drive%2:38:00::","synset_id":"drive.v.02"},{"index":3,"sense_key":"drive%2:41:13::","synset_id":"drive.v.13"},{"index":4,"sense_key":"drive%2:38:11::","synset_id":"drive.v.16"},{"index":5,"sense_key":"drive%2:32:00::","synset_id":"drive.v.11"},{"index":6,"sense_key":"drive%2:33:00::","synset_id":"drive.v.22"},{"index":7,"sense_key":"drive%2:35:02::","synset_id":"drive.v.18"},{"index":8,"sense_key":"drive%2:35:06::","synset_id":"drive.v.19"},{"index":9,"sense_key":"drive%2:38:10::","synset_id":"drive.v.15"},{"index":10,"sense_key":"drive%2:42:00::","synset_id":"drive.v.12"},{"index":11,"sense_key":"drive%2:41:02::","synset_id":"drive.v.05"},{"index":12,"sense_key":"drive%2:35:11::","synset_id":"repel.v.01"},{"index":13,"sense_key":"drive%2:35:07::","synset_id":"drive.v.08"},{"index":14,"sense_key":"drive%2:38:09::","synset_id":"drive.v.14"},{"index":15,"sense_key":"drive%2:34:07::","synset_id":"drive.v.20"},{"index":16,"sense_key":"drive%2:35:00::","synset_id":"force.v.06"},{"index":17,"sense_key":"drive%2:38:01::","synset_id":"drive.v.01"},{"index":18,"sense_key":"drive%2:35:03::","synset_id":"drive.v.17"},{"index":19,"sense_key":"drive%2:38:02::","synset_id":"drive.v.03"},{"index":20,"sense_key":"drive%2:41:00::","synset_id":"tug.v.02"},{"index":21,"sense_key":"drive%2:36:00::","synset_id":"drive.v.07"},{"index":22,"sense_key":"drive%2:33:01::","synset_id":"drive.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"drive%2:38:01::","call_ids":["senseval3.d002.s040.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"drive%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1116,"cached_input_tokens":0,"output_tokens":175,"reasoning_output_tokens":159},"cost":{"currency":"USD","total_usd":0.010830000000000001,"input_uncached_usd":0.005580000000000001,"input_cached_usd":0.0,"output_usd":0.00525,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.53077904204838}
{"item_id":"senseval3.d002.s040.t001","gold_sense_keys":["truck%1:06:00::"],"candidates":[{"index":1,"sense_key":"truck%1:06:01::","synset_id":"hand_truck.n.01"},{"index":2,"sense_key":"truck%1:06:00::","synset_id":"truck.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"truck%1:06:00::","call_ids":["senseval3.d002.s040.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"truck%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":334,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00359,"input_uncached_usd":0.00167,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2783152089687064}
{"item_id":"senseval3.d002.s040.t002","gold_sense_keys":["stop%2:38:00::"],"candidates":[{"index":1,"sense_key":"stop%2:35:00::","synset_id":"intercept.v.01"},{"index":2,"sense_key":"stop%2:38:02::","synset_id":"stop.v.04"},{"index":3,"sense_key":"stop%2:38:00::","synset_id":"stop.v.01"},{"index":4,"sense_key":"stop%2:38:01::","synset_id":"stop.v.05"},{"index":5,"sense_key":"stop%2:30:12::","synset_id":"hold_on.v.02"},{"index":6,"sense_key":"stop%2:42:00::","synset_id":"discontinue.v.01"},{"index":7,"sense_key":"stop%2:41:00::","synset_id":"stop.v.03"},{"index":8,"sense_key":"stop%2:33:00::","synset_id":"check.v.18"},{"index":9,"sense_key":"stop%2:30:05::","synset_id":"break.v.10"},{"index":10,"sense_key":"stop%2:42:13::","synset_id":"end.v.01"},{"index":11,"sense_key":"stop%2:35:12::","synset_id":"barricade.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"stop%2:38:00::","call_ids":["senseval3.d002.s040.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"stop%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":220,"reasoning_output_tokens":204},"cost":{"currency":"USD","total_usd":0.010465000000000002,"input_uncached_usd":0.0038650000000000004,"input_cached_usd":0.0,"output_usd":0.0066,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.353807333041914}
{"item_id":"senseval3.d002.s040.t003","gold_sense_keys":["red_light%1:10:00::"],"candidates":[{"index":1,"sense_key":"red_light%1:10:00::","synset_id":"red_light.n.02"},{"index":2,"sense_key":"red_light%1:10:01::","synset_id":"red_light.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"red_light%1:10:00::","call_ids":["senseval3.d002.s040.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"red_light%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":306,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.00336,"input_uncached_usd":0.0015300000000000001,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3784830829827115}
{"item_id":"senseval3.d002.s040.t004","gold_sense_keys":["corner%1:06:00::"],"candidates":[{"index":1,"sense_key":"corner%1:26:01::","synset_id":"corner.n.08"},{"index":2,"sense_key":"corner%1:06:02::","synset_id":"corner.n.03"},{"index":3,"sense_key":"corner%1:06:00::","synset_id":"corner.n.04"},{"index":4,"sense_key":"corner%1:17:00::","synset_id":"corner.n.09"},{"index":5,"sense_key":"corner%1:25:00::","synset_id":"recess.n.02"},{"index":6,"sense_key":"corner%1:15:02::","synset_id":"corner.n.01"},{"index":7,"sense_key":"corner%1:06:01::","synset_id":"corner.n.11"},{"index":8,"sense_key":"corner%1:15:03::","synset_id":"corner.n.10"},{"index":9,"sense_key":"corner%1:15:01::","synset_id":"corner.n.05"},{"index":10,"sense_key":"corner%1:26:00::","synset_id":"corner.n.07"},{"index":11,"sense_key":"corner%1:15:00::","synset_id":"corner.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"corner%1:06:00::","call_ids":["senseval3.d002.s040.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"corner%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.005575,"input_uncached_usd":0.003265,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8430779160698876}
{"item_id":"senseval3.d002.s040.t006","gold_sense_keys":["hit%2:35:05::"],"candidates":[{"index":1,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"},{"index":2,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"},{"index":3,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":4,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"},{"index":5,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":6,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":7,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"},{"index":8,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"},{"index":9,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":10,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"},{"index":11,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":12,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":13,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":14,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":15,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":16,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":17,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"hit%2:35:05::","call_ids":["senseval3.d002.s040.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"hit%2:35:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":991,"cached_input_tokens":0,"output_tokens":248,"reasoning_output_tokens":233},"cost":{"currency":"USD","total_usd":0.012395,"input_uncached_usd":0.004955,"input_cached_usd":0.0,"output_usd":0.00744,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.269715833012015}
{"item_id":"senseval3.d002.s041.t000","gold_sense_keys":["part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":2,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":5,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":6,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"},{"index":7,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":8,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":11,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":12,"sense_key":"part%1:04:01::","synset_id":"function.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"part%1:09:00::","call_ids":["senseval3.d002.s041.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"part%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":842,"cached_input_tokens":0,"output_tokens":319,"reasoning_output_tokens":303},"cost":{"currency":"USD","total_usd":0.01378,"input_uncached_usd":0.00421,"input_cached_usd":0.0,"output_usd":0.00957,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.660320875002071}
{"item_id":"senseval3.d002.s041.t001","gold_sense_keys":["watch%2:39:00::","watch%2:39:02::","watch%2:41:00::"],"candidates":[{"index":1,"sense_key":"watch%2:32:09::","synset_id":"determine.v.08"},{"index":2,"sense_key":"watch%2:39:00::","synset_id":"watch.v.01"},{"index":3,"sense_key":"watch%2:39:01::","synset_id":"watch.v.05"},{"index":4,"sense_key":"watch%2:39:05::","synset_id":"watch.v.03"},{"index":5,"sense_key":"watch%2:39:02::","synset_id":"watch.v.04"},{"index":6,"sense_key":"watch%2:39:03::","synset_id":"watch.v.06"},{"index":7,"sense_key":"watch%2:41:00::","synset_id":"watch.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"watch%2:39:02::","call_ids":["senseval3.d002.s041.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"watch%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":1027,"reasoning_output_tokens":1005},"cost":{"currency":"USD","total_usd":0.033605,"input_uncached_usd":0.002795,"input_cached_usd":0.0,"output_usd":0.03081,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":17.95146625000052}
{"item_id":"senseval3.d002.s041.t003","gold_sense_keys":["wave%2:38:00::"],"candidates":[{"index":1,"sense_key":"wave%2:35:00::","synset_id":"brandish.v.01"},{"index":2,"sense_key":"wave%2:32:00::","synset_id":"beckon.v.01"},{"index":3,"sense_key":"wave%2:35:01::","synset_id":"curl.v.04"},{"index":4,"sense_key":"wave%2:29:00::","synset_id":"wave.v.05"},{"index":5,"sense_key":"wave%2:38:00::","synset_id":"roll.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"wave%2:38:00::","call_ids":["senseval3.d002.s041.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"wave%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0181,"input_uncached_usd":0.0022600000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.584668707917444}
{"item_id":"senseval3.d002.s041.t004","gold_sense_keys":["head%1:08:00::"],"candidates":[{"index":1,"sense_key":"head%1:17:02::","synset_id":"promontory.n.01"},{"index":2,"sense_key":"head%1:06:01::","synset_id":"drumhead.n.01"},{"index":3,"sense_key":"head%1:08:00::","synset_id":"head.n.01"},{"index":4,"sense_key":"head%1:15:01::","synset_id":"fountainhead.n.02"},{"index":5,"sense_key":"head%1:17:01::","synset_id":"head.n.17"},{"index":6,"sense_key":"head%1:15:02::","synset_id":"forefront.n.01"},{"index":7,"sense_key":"head%1:04:00::","synset_id":"oral_sex.n.01"},{"index":8,"sense_key":"head%1:26:00::","synset_id":"head.n.10"},{"index":9,"sense_key":"head%1:18:00::","synset_id":"head.n.04"},{"index":10,"sense_key":"head%1:23:00::","synset_id":"head.n.11"},{"index":11,"sense_key":"head%1:20:00::","synset_id":"capitulum.n.01"},{"index":12,"sense_key":"head%1:10:02::","synset_id":"question.n.02"},{"index":13,"sense_key":"head%1:15:00::","synset_id":"head.n.07"},{"index":14,"sense_key":"head%1:09:00::","synset_id":"mind.n.01"},{"index":15,"sense_key":"head%1:18:01::","synset_id":"principal.n.02"},{"index":16,"sense_key":"head%1:06:03::","synset_id":"head.n.29"},{"index":17,"sense_key":"head%1:14:00::","synset_id":"head.n.05"},{"index":18,"sense_key":"head%1:18:02::","synset_id":"head.n.15"},{"index":19,"sense_key":"head%1:17:00::","synset_id":"head.n.18"},{"index":20,"sense_key":"head%1:05:00::","synset_id":"head.n.02"},{"index":21,"sense_key":"head%1:10:00::","synset_id":"heading.n.01"},{"index":22,"sense_key":"head%1:08:01::","synset_id":"head.n.25"},{"index":23,"sense_key":"head%1:11:01::","synset_id":"pass.n.09"},{"index":24,"sense_key":"head%1:11:00::","synset_id":"headway.n.02"},{"index":25,"sense_key":"head%1:19:00::","synset_id":"head.n.06"},{"index":26,"sense_key":"head%1:06:02::","synset_id":"head.n.30"},{"index":27,"sense_key":"head%1:06:05::","synset_id":"read/write_head.n.01"},{"index":28,"sense_key":"head%1:10:03::","synset_id":"head.n.09"},{"index":29,"sense_key":"head%1:18:03::","synset_id":"head.n.14"},{"index":30,"sense_key":"head%1:08:02::","synset_id":"head.n.26"},{"index":31,"sense_key":"head%1:10:01::","synset_id":"point.n.20"},{"index":32,"sense_key":"head%1:06:04::","synset_id":"head.n.28"},{"index":33,"sense_key":"head%1:06:00::","synset_id":"head.n.31"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"head%1:08:00::","call_ids":["senseval3.d002.s041.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"head%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1533,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.009135,"input_uncached_usd":0.007665000000000001,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0061926250346005}
{"item_id":"senseval3.d002.s041.t005","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":2,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":3,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":4,"sense_key":"way%1:21:00::","synset_id":"way.n.12"},{"index":5,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":6,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":7,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":8,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":9,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":10,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":11,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":12,"sense_key":"way%1:04:02::","synset_id":"way.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"way%1:04:01::","call_ids":["senseval3.d002.s041.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":720,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01944,"input_uncached_usd":0.0036000000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.363399666035548}
{"item_id":"senseval3.d002.s041.t006","gold_sense_keys":["drive%2:38:00::"],"candidates":[{"index":1,"sense_key":"drive%2:35:11::","synset_id":"repel.v.01"},{"index":2,"sense_key":"drive%2:33:00::","synset_id":"drive.v.22"},{"index":3,"sense_key":"drive%2:38:09::","synset_id":"drive.v.14"},{"index":4,"sense_key":"drive%2:35:06::","synset_id":"drive.v.19"},{"index":5,"sense_key":"drive%2:38:00::","synset_id":"drive.v.02"},{"index":6,"sense_key":"drive%2:35:02::","synset_id":"drive.v.18"},{"index":7,"sense_key":"drive%2:42:00::","synset_id":"drive.v.12"},{"index":8,"sense_key":"drive%2:35:07::","synset_id":"drive.v.08"},{"index":9,"sense_key":"drive%2:41:00::","synset_id":"tug.v.02"},{"index":10,"sense_key":"drive%2:38:02::","synset_id":"drive.v.03"},{"index":11,"sense_key":"drive%2:38:01::","synset_id":"drive.v.01"},{"index":12,"sense_key":"drive%2:35:03::","synset_id":"drive.v.17"},{"index":13,"sense_key":"drive%2:41:02::","synset_id":"drive.v.05"},{"index":14,"sense_key":"drive%2:32:00::","synset_id":"drive.v.11"},{"index":15,"sense_key":"drive%2:35:01::","synset_id":"drive.v.09"},{"index":16,"sense_key":"drive%2:35:00::","synset_id":"force.v.06"},{"index":17,"sense_key":"drive%2:36:00::","synset_id":"drive.v.07"},{"index":18,"sense_key":"drive%2:38:11::","synset_id":"drive.v.16"},{"index":19,"sense_key":"drive%2:38:10::","synset_id":"drive.v.15"},{"index":20,"sense_key":"drive%2:41:13::","synset_id":"drive.v.13"},{"index":21,"sense_key":"drive%2:33:01::","synset_id":"drive.v.21"},{"index":22,"sense_key":"drive%2:34:07::","synset_id":"drive.v.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"drive%2:38:00::","call_ids":["senseval3.d002.s041.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"drive%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1110,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02139,"input_uncached_usd":0.00555,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.630205541965552}
{"item_id":"senseval3.d002.s044.t000","gold_sense_keys":["foot%1:23:00::"],"candidates":[{"index":1,"sense_key":"foot%1:06:01::","synset_id":"foot.n.11"},{"index":2,"sense_key":"foot%1:23:00::","synset_id":"foot.n.02"},{"index":3,"sense_key":"foot%1:04:00::","synset_id":"foot.n.07"},{"index":4,"sense_key":"foot%1:15:00::","synset_id":"foot.n.03"},{"index":5,"sense_key":"foot%1:05:01::","synset_id":"animal_foot.n.01"},{"index":6,"sense_key":"foot%1:06:00::","synset_id":"foundation.n.03"},{"index":7,"sense_key":"foot%1:18:00::","synset_id":"foot.n.08"},{"index":8,"sense_key":"foot%1:08:01::","synset_id":"foot.n.01"},{"index":9,"sense_key":"foot%1:10:00::","synset_id":"metrical_foot.n.01"},{"index":10,"sense_key":"foot%1:05:00::","synset_id":"foot.n.06"},{"index":11,"sense_key":"foot%1:14:00::","synset_id":"infantry.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"foot%1:23:00::","call_ids":["senseval3.d002.s044.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"foot%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.005105,"input_uncached_usd":0.003455,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5320842920336872}
{"item_id":"senseval3.d002.s044.t001","gold_sense_keys":["pier%1:06:00::"],"candidates":[{"index":1,"sense_key":"pier%1:06:01::","synset_id":"pier.n.03"},{"index":2,"sense_key":"pier%1:06:02::","synset_id":"pier.n.02"},{"index":3,"sense_key":"pier%1:06:00::","synset_id":"pier.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"pier%1:06:00::","call_ids":["senseval3.d002.s044.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"pier%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":339,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.003855,"input_uncached_usd":0.0016950000000000001,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8683764579473063}
{"item_id":"senseval3.d002.s045.t001","gold_sense_keys":["dramatic%5:00:00:impressive:00"],"candidates":[{"index":1,"sense_key":"dramatic%3:01:00::","synset_id":"dramatic.a.03"},{"index":2,"sense_key":"dramatic%5:00:00:impressive:00","synset_id":"dramatic.s.01"},{"index":3,"sense_key":"dramatic%3:00:02::","synset_id":"dramatic.a.04"},{"index":4,"sense_key":"dramatic%3:00:00::","synset_id":"dramatic.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dramatic%5:00:00:impressive:00","call_ids":["senseval3.d002.s045.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dramatic%5:00:00:impressive:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.004445000000000001,"input_uncached_usd":0.0020450000000000004,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9532009579706937}
{"item_id":"senseval3.d002.s046.t001","gold_sense_keys":["hairline%1:10:00::"],"candidates":[{"index":1,"sense_key":"hairline%1:08:00::","synset_id":"hairline.n.02"},{"index":2,"sense_key":"hairline%1:10:00::","synset_id":"hairline.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"hairline%1:10:00::","call_ids":["senseval3.d002.s046.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"hairline%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":265,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.002765,"input_uncached_usd":0.001325,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.559044791967608}
{"item_id":"senseval3.d002.s046.t002","gold_sense_keys":["crack%1:07:00::"],"candidates":[{"index":1,"sense_key":"crack%1:17:00::","synset_id":"crack.n.01"},{"index":2,"sense_key":"crack%1:07:00::","synset_id":"crack.n.07"},{"index":3,"sense_key":"crack%1:04:00::","synset_id":"fracture.n.03"},{"index":4,"sense_key":"crack%1:06:01::","synset_id":"crack.n.08"},{"index":5,"sense_key":"crack%1:10:00::","synset_id":"wisecrack.n.01"},{"index":6,"sense_key":"crack%1:11:00::","synset_id":"crack.n.04"},{"index":7,"sense_key":"crack%1:06:00::","synset_id":"gap.n.03"},{"index":8,"sense_key":"crack%1:26:00::","synset_id":"shot.n.04"},{"index":9,"sense_key":"crack%1:25:00::","synset_id":"crevice.n.01"},{"index":10,"sense_key":"crack%1:04:01::","synset_id":"crack.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"crack%1:07:00::","call_ids":["senseval3.d002.s046.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"crack%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":194,"reasoning_output_tokens":178},"cost":{"currency":"USD","total_usd":0.008795,"input_uncached_usd":0.002975,"input_cached_usd":0.0,"output_usd":0.0058200000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3685875840019435}
{"item_id":"senseval3.d002.s047.t000","gold_sense_keys":["ruin%2:41:00::"],"candidates":[{"index":1,"sense_key":"ruin%2:35:00::","synset_id":"destroy.v.02"},{"index":2,"sense_key":"ruin%2:40:00::","synset_id":"bankrupt.v.01"},{"index":3,"sense_key":"ruin%2:41:00::","synset_id":"ruin.v.02"},{"index":4,"sense_key":"ruin%2:30:00::","synset_id":"ruin.v.06"},{"index":5,"sense_key":"ruin%2:35:01::","synset_id":"ruin.v.04"},{"index":6,"sense_key":"ruin%2:35:02::","synset_id":"deflower.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ruin%2:41:00::","call_ids":["senseval3.d002.s047.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ruin%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":457,"reasoning_output_tokens":442},"cost":{"currency":"USD","total_usd":0.015915000000000002,"input_uncached_usd":0.0022050000000000004,"input_cached_usd":0.0,"output_usd":0.01371,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.206084334058687}
{"item_id":"senseval3.d002.s047.t001","gold_sense_keys":["damn%5:00:02:cursed:00"],"candidates":[{"index":1,"sense_key":"damn%5:00:00:cursed:00","synset_id":"damn.s.01"},{"index":2,"sense_key":"damn%5:00:02:cursed:00","synset_id":"blasted.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"damn%5:00:02:cursed:00","call_ids":["senseval3.d002.s047.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"damn%5:00:02:cursed:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":307,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.0031850000000000003,"input_uncached_usd":0.0015350000000000001,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.487027750001289}
{"item_id":"senseval3.d002.s050.t001","gold_sense_keys":["second%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"second%5:00:00:ordinal:00","synset_id":"second.s.01"},{"index":2,"sense_key":"second%3:00:00::","synset_id":"second.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%5:00:00:ordinal:00","call_ids":["senseval3.d002.s050.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":303,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.002985,"input_uncached_usd":0.001515,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.148288166965358}
{"item_id":"senseval3.d002.s050.t002","gold_sense_keys":["floor%1:06:01::"],"candidates":[{"index":1,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":2,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"},{"index":3,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"},{"index":4,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":5,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":6,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"},{"index":7,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":8,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"},{"index":9,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"},{"index":10,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"floor%1:06:01::","call_ids":["senseval3.d002.s050.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"floor%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.005495000000000001,"input_uncached_usd":0.0030650000000000004,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.605525541002862}
{"item_id":"senseval3.d002.s051.t000","gold_sense_keys":["hear%2:39:00::"],"candidates":[{"index":1,"sense_key":"hear%2:39:05::","synset_id":"listen.v.02"},{"index":2,"sense_key":"hear%2:31:00::","synset_id":"learn.v.02"},{"index":3,"sense_key":"hear%2:39:01::","synset_id":"hear.v.04"},{"index":4,"sense_key":"hear%2:41:00::","synset_id":"hear.v.03"},{"index":5,"sense_key":"hear%2:39:00::","synset_id":"hear.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"hear%2:39:00::","call_ids":["senseval3.d002.s051.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"hear%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.00382,"input_uncached_usd":0.0022,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.329591333982535}
{"item_id":"senseval3.d002.s051.t001","gold_sense_keys":["part%1:06:00::","part%1:15:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":3,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":4,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"},{"index":5,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":6,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":7,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":8,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":9,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":10,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":11,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":12,"sense_key":"part%1:09:01::","synset_id":"character.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"part%1:06:00::","call_ids":["senseval3.d002.s051.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"part%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":807,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":101},"cost":{"currency":"USD","total_usd":0.007515000000000001,"input_uncached_usd":0.004035,"input_cached_usd":0.0,"output_usd":0.00348,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7354562500258908}
{"item_id":"senseval3.d002.s051.t002","gold_sense_keys":["building%1:06:00::"],"candidates":[{"index":1,"sense_key":"building%1:14:00::","synset_id":"building.n.04"},{"index":2,"sense_key":"building%1:04:00::","synset_id":"construction.n.01"},{"index":3,"sense_key":"building%1:06:00::","synset_id":"building.n.01"},{"index":4,"sense_key":"building%1:04:01::","synset_id":"construction.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"building%1:06:00::","call_ids":["senseval3.d002.s051.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"building%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":398,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.0034000000000000002,"input_uncached_usd":0.00199,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5749182080617175}
{"item_id":"senseval3.d002.s051.t003","gold_sense_keys":["head%1:08:00::"],"candidates":[{"index":1,"sense_key":"head%1:11:00::","synset_id":"headway.n.02"},{"index":2,"sense_key":"head%1:19:00::","synset_id":"head.n.06"},{"index":3,"sense_key":"head%1:06:00::","synset_id":"head.n.31"},{"index":4,"sense_key":"head%1:18:02::","synset_id":"head.n.15"},{"index":5,"sense_key":"head%1:23:00::","synset_id":"head.n.11"},{"index":6,"sense_key":"head%1:20:00::","synset_id":"capitulum.n.01"},{"index":7,"sense_key":"head%1:17:01::","synset_id":"head.n.17"},{"index":8,"sense_key":"head%1:10:00::","synset_id":"heading.n.01"},{"index":9,"sense_key":"head%1:05:00::","synset_id":"head.n.02"},{"index":10,"sense_key":"head%1:15:00::","synset_id":"head.n.07"},{"index":11,"sense_key":"head%1:06:03::","synset_id":"head.n.29"},{"index":12,"sense_key":"head%1:17:00::","synset_id":"head.n.18"},{"index":13,"sense_key":"head%1:18:01::","synset_id":"principal.n.02"},{"index":14,"sense_key":"head%1:14:00::","synset_id":"head.n.05"},{"index":15,"sense_key":"head%1:17:02::","synset_id":"promontory.n.01"},{"index":16,"sense_key":"head%1:04:00::","synset_id":"oral_sex.n.01"},{"index":17,"sense_key":"head%1:10:01::","synset_id":"point.n.20"},{"index":18,"sense_key":"head%1:26:00::","synset_id":"head.n.10"},{"index":19,"sense_key":"head%1:15:01::","synset_id":"fountainhead.n.02"},{"index":20,"sense_key":"head%1:10:03::","synset_id":"head.n.09"},{"index":21,"sense_key":"head%1:06:01::","synset_id":"drumhead.n.01"},{"index":22,"sense_key":"head%1:08:00::","synset_id":"head.n.01"},{"index":23,"sense_key":"head%1:06:04::","synset_id":"head.n.28"},{"index":24,"sense_key":"head%1:06:05::","synset_id":"read/write_head.n.01"},{"index":25,"sense_key":"head%1:09:00::","synset_id":"mind.n.01"},{"index":26,"sense_key":"head%1:11:01::","synset_id":"pass.n.09"},{"index":27,"sense_key":"head%1:08:01::","synset_id":"head.n.25"},{"index":28,"sense_key":"head%1:18:03::","synset_id":"head.n.14"},{"index":29,"sense_key":"head%1:15:02::","synset_id":"forefront.n.01"},{"index":30,"sense_key":"head%1:06:02::","synset_id":"head.n.30"},{"index":31,"sense_key":"head%1:18:00::","synset_id":"head.n.04"},{"index":32,"sense_key":"head%1:08:02::","synset_id":"head.n.26"},{"index":33,"sense_key":"head%1:10:02::","synset_id":"question.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":22,"chosen_sense_key":"head%1:08:00::","call_ids":["senseval3.d002.s051.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":22,"predicted_sense_key":"head%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1498,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.00884,"input_uncached_usd":0.007490000000000001,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6249022090341896}
{"item_id":"senseval3.d002.s051.t004","gold_sense_keys":["crack%2:30:00::","crack%2:30:01::","crack%2:39:00::","crack%2:39:01::"],"candidates":[{"index":1,"sense_key":"crack%2:30:02::","synset_id":"crack.v.06"},{"index":2,"sense_key":"crack%2:30:00::","synset_id":"snap.v.03"},{"index":3,"sense_key":"crack%2:30:01::","synset_id":"crack.v.01"},{"index":4,"sense_key":"crack%2:35:00::","synset_id":"crack.v.04"},{"index":5,"sense_key":"crack%2:39:00::","synset_id":"snap.v.06"},{"index":6,"sense_key":"crack%2:41:00::","synset_id":"crack.v.08"},{"index":7,"sense_key":"crack%2:37:00::","synset_id":"crack_up.v.01"},{"index":8,"sense_key":"crack%2:30:08::","synset_id":"crack.v.13"},{"index":9,"sense_key":"crack%2:30:05::","synset_id":"crack.v.11"},{"index":10,"sense_key":"crack%2:39:01::","synset_id":"crack.v.02"},{"index":11,"sense_key":"crack%2:30:09::","synset_id":"break_through.v.01"},{"index":12,"sense_key":"crack%2:32:00::","synset_id":"crack.v.10"},{"index":13,"sense_key":"crack%2:30:07::","synset_id":"crack.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crack%2:30:02::","call_ids":["senseval3.d002.s051.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crack%2:30:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":671,"cached_input_tokens":0,"output_tokens":8460,"reasoning_output_tokens":8346},"cost":{"currency":"USD","total_usd":0.25715499999999997,"input_uncached_usd":0.0033550000000000003,"input_cached_usd":0.0,"output_usd":0.2538,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":136.4966494999826}
{"item_id":"senseval3.d002.s052.t000","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:30:00::","synset_id":"think.v.13"},{"index":2,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":3,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":4,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":5,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":6,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":7,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":8,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":9,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":10,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":13,"sense_key":"think%2:31:01::","synset_id":"think.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval3.d002.s052.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":791,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019795,"input_uncached_usd":0.003955,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.947423040983267}
{"item_id":"senseval3.d002.s052.t001","gold_sense_keys":["die%2:30:00::"],"candidates":[{"index":1,"sense_key":"die%2:35:00::","synset_id":"die.v.07"},{"index":2,"sense_key":"die%2:37:03::","synset_id":"die.v.05"},{"index":3,"sense_key":"die%2:37:01::","synset_id":"die.v.06"},{"index":4,"sense_key":"die%2:39:00::","synset_id":"die.v.02"},{"index":5,"sense_key":"die%2:30:00::","synset_id":"die.v.01"},{"index":6,"sense_key":"die%2:30:05::","synset_id":"die.v.09"},{"index":7,"sense_key":"die%2:33:00::","synset_id":"die.v.08"},{"index":8,"sense_key":"die%2:30:02::","synset_id":"die.v.11"},{"index":9,"sense_key":"die%2:30:04::","synset_id":"fail.v.04"},{"index":10,"sense_key":"die%2:30:01::","synset_id":"die.v.10"},{"index":11,"sense_key":"die%2:37:00::","synset_id":"die.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"die%2:30:00::","call_ids":["senseval3.d002.s052.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"die%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.006065,"input_uncached_usd":0.003455,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.742714916006662}
{"item_id":"senseval3.d002.s053.t000","gold_sense_keys":["decide%2:31:00::"],"candidates":[{"index":1,"sense_key":"decide%2:31:06::","synset_id":"decide.v.04"},{"index":2,"sense_key":"decide%2:31:02::","synset_id":"decide.v.03"},{"index":3,"sense_key":"decide%2:31:00::","synset_id":"decide.v.01"},{"index":4,"sense_key":"decide%2:31:01::","synset_id":"decide.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"decide%2:31:00::","call_ids":["senseval3.d002.s053.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"decide%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":393,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.003585,"input_uncached_usd":0.001965,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3394916669931263}
{"item_id":"senseval3.d002.s053.t001","gold_sense_keys":["come%2:38:00::"],"candidates":[{"index":1,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":2,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":3,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":4,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":5,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":6,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":7,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":8,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":11,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":12,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":13,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":14,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":15,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":16,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":17,"sense_key":"come%2:31:13::","synset_id":"come.v.21"},{"index":18,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":19,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":20,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":21,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":21,"chosen_sense_key":"come%2:38:04::","call_ids":["senseval3.d002.s053.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":21,"predicted_sense_key":"come%2:38:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1096,"cached_input_tokens":0,"output_tokens":3715,"reasoning_output_tokens":3658},"cost":{"currency":"USD","total_usd":0.11693,"input_uncached_usd":0.0054800000000000005,"input_cached_usd":0.0,"output_usd":0.11145000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":81.01470275002066}
{"item_id":"senseval3.d002.s053.t002","gold_sense_keys":["home%1:15:04::"],"candidates":[{"index":1,"sense_key":"home%1:15:02::","synset_id":"home.n.03"},{"index":2,"sense_key":"home%1:14:00::","synset_id":"family.n.01"},{"index":3,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"},{"index":4,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":5,"sense_key":"home%1:15:03::","synset_id":"home.n.06"},{"index":6,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":7,"sense_key":"home%1:06:01::","synset_id":"home.n.09"},{"index":8,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":9,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"home%1:15:04::","call_ids":["senseval3.d002.s053.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"home%1:15:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01902,"input_uncached_usd":0.00318,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.999432957964018}
{"item_id":"senseval3.d002.s053.t003","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":2,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":3,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":4,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":5,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":6,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":7,"sense_key":"house%1:06:01::","synset_id":"house.n.12"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":10,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":11,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":12,"sense_key":"house%1:14:03::","synset_id":"house.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s053.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.0049250000000000006,"input_uncached_usd":0.0034850000000000003,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4756835419684649}
{"item_id":"senseval3.d002.s053.t004","gold_sense_keys":["stilt%1:06:01::"],"candidates":[{"index":1,"sense_key":"stilt%1:05:01::","synset_id":"stilt.n.04"},{"index":2,"sense_key":"stilt%1:05:02::","synset_id":"stilt.n.03"},{"index":3,"sense_key":"stilt%1:06:01::","synset_id":"pile.n.06"},{"index":4,"sense_key":"stilt%1:06:00::","synset_id":"stilt.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"stilt%1:06:01::","call_ids":["senseval3.d002.s053.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"stilt%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.0037,"input_uncached_usd":0.0021100000000000003,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2398971250513569}
{"item_id":"senseval3.d002.s054.t000","gold_sense_keys":["decide%2:31:00::"],"candidates":[{"index":1,"sense_key":"decide%2:31:01::","synset_id":"decide.v.02"},{"index":2,"sense_key":"decide%2:31:06::","synset_id":"decide.v.04"},{"index":3,"sense_key":"decide%2:31:02::","synset_id":"decide.v.03"},{"index":4,"sense_key":"decide%2:31:00::","synset_id":"decide.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"decide%2:31:00::","call_ids":["senseval3.d002.s054.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"decide%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.003935000000000001,"input_uncached_usd":0.0020450000000000004,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0244136248948053}
{"item_id":"senseval3.d002.s055.t001","gold_sense_keys":["smell%1:07:00::","smell%1:09:02::"],"candidates":[{"index":1,"sense_key":"smell%1:26:00::","synset_id":"spirit.n.02"},{"index":2,"sense_key":"smell%1:04:00::","synset_id":"smell.n.05"},{"index":3,"sense_key":"smell%1:09:01::","synset_id":"smell.n.04"},{"index":4,"sense_key":"smell%1:07:00::","synset_id":"olfactory_property.n.01"},{"index":5,"sense_key":"smell%1:09:02::","synset_id":"smell.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"smell%1:09:02::","call_ids":["senseval3.d002.s055.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"smell%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018349999999999998,"input_uncached_usd":0.00251,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.380588709027506}
{"item_id":"senseval3.d002.s055.t002","gold_sense_keys":["gas%1:27:02::"],"candidates":[{"index":1,"sense_key":"gas%1:26:00::","synset_id":"gas.n.01"},{"index":2,"sense_key":"gas%1:27:00::","synset_id":"gas.n.02"},{"index":3,"sense_key":"gas%1:27:02::","synset_id":"gasoline.n.01"},{"index":4,"sense_key":"gas%1:26:01::","synset_id":"flatulence.n.01"},{"index":5,"sense_key":"gas%1:06:00::","synset_id":"accelerator.n.01"},{"index":6,"sense_key":"gas%1:27:03::","synset_id":"natural_gas.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"gas%1:27:00::","call_ids":["senseval3.d002.s055.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"gas%1:27:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":542,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01855,"input_uncached_usd":0.00271,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.236309499945492}
{"item_id":"senseval3.d002.s055.t003","gold_sense_keys":["pass%2:38:05::"],"candidates":[{"index":1,"sense_key":"pass%2:42:00::","synset_id":"spend.v.01"},{"index":2,"sense_key":"pass%2:29:00::","synset_id":"excrete.v.01"},{"index":3,"sense_key":"pass%2:42:04::","synset_id":"exceed.v.02"},{"index":4,"sense_key":"pass%2:35:00::","synset_id":"guide.v.05"},{"index":5,"sense_key":"pass%2:38:03::","synset_id":"elapse.v.01"},{"index":6,"sense_key":"pass%2:38:05::","synset_id":"travel_by.v.01"},{"index":7,"sense_key":"pass%2:40:02::","synset_id":"pass.v.18"},{"index":8,"sense_key":"pass%2:38:00::","synset_id":"pass.v.01"},{"index":9,"sense_key":"pass%2:42:08::","synset_id":"run.v.03"},{"index":10,"sense_key":"pass%2:41:11::","synset_id":"pass.v.17"},{"index":11,"sense_key":"pass%2:40:00::","synset_id":"pass.v.05"},{"index":12,"sense_key":"pass%2:32:01::","synset_id":"communicate.v.01"},{"index":13,"sense_key":"pass%2:30:11::","synset_id":"die.v.01"},{"index":14,"sense_key":"pass%2:30:00::","synset_id":"happen.v.01"},{"index":15,"sense_key":"pass%2:41:04::","synset_id":"pass.v.09"},{"index":16,"sense_key":"pass%2:38:02::","synset_id":"sink.v.03"},{"index":17,"sense_key":"pass%2:40:01::","synset_id":"pass.v.20"},{"index":18,"sense_key":"pass%2:30:01::","synset_id":"evanesce.v.01"},{"index":19,"sense_key":"pass%2:41:08::","synset_id":"pass.v.14"},{"index":20,"sense_key":"pass%2:41:02::","synset_id":"legislate.v.01"},{"index":21,"sense_key":"pass%2:41:07::","synset_id":"pass.v.16"},{"index":22,"sense_key":"pass%2:38:04::","synset_id":"pass.v.22"},{"index":23,"sense_key":"pass%2:32:00::","synset_id":"authorize.v.01"},{"index":24,"sense_key":"pass%2:38:01::","synset_id":"pass.v.07"},{"index":25,"sense_key":"pass%2:40:13::","synset_id":"fall.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"pass%2:38:05::","call_ids":["senseval3.d002.s055.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"pass%2:38:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1395,"cached_input_tokens":0,"output_tokens":504,"reasoning_output_tokens":488},"cost":{"currency":"USD","total_usd":0.022095,"input_uncached_usd":0.006975,"input_cached_usd":0.0,"output_usd":0.01512,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.552018792019226}
{"item_id":"senseval3.d002.s055.t005","gold_sense_keys":["cross%2:38:00::"],"candidates":[{"index":1,"sense_key":"cross%2:38:04::","synset_id":"cross.v.04"},{"index":2,"sense_key":"cross%2:36:09::","synset_id":"cross.v.07"},{"index":3,"sense_key":"cross%2:35:00::","synset_id":"crossbreed.v.01"},{"index":4,"sense_key":"cross%2:38:03::","synset_id":"intersect.v.01"},{"index":5,"sense_key":"cross%2:42:00::","synset_id":"cross.v.05"},{"index":6,"sense_key":"cross%2:38:01::","synset_id":"cross.v.06"},{"index":7,"sense_key":"cross%2:41:00::","synset_id":"thwart.v.01"},{"index":8,"sense_key":"cross%2:38:00::","synset_id":"traverse.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"cross%2:38:00::","call_ids":["senseval3.d002.s055.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"cross%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":600,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":94},"cost":{"currency":"USD","total_usd":0.0063,"input_uncached_usd":0.003,"input_cached_usd":0.0,"output_usd":0.0033,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1062775829341263}
{"item_id":"senseval3.d002.s056.t000","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":2,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":3,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":4,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":5,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":6,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":7,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":8,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":11,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":14,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":15,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":16,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":17,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":18,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":19,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":20,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":21,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":22,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":23,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":24,"sense_key":"see%2:33:00::","synset_id":"see.v.23"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval3.d002.s056.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1388,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.00901,"input_uncached_usd":0.006940000000000001,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.79456095804926}
{"item_id":"senseval3.d002.s056.t001","gold_sense_keys":["cloud%1:19:01::"],"candidates":[{"index":1,"sense_key":"cloud%1:14:00::","synset_id":"swarm.n.02"},{"index":2,"sense_key":"cloud%1:19:01::","synset_id":"cloud.n.01"},{"index":3,"sense_key":"cloud%1:26:02::","synset_id":"cloud.n.04"},{"index":4,"sense_key":"cloud%1:26:01::","synset_id":"cloud.n.03"},{"index":5,"sense_key":"cloud%1:26:00::","synset_id":"cloud.n.05"},{"index":6,"sense_key":"cloud%1:17:00::","synset_id":"cloud.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cloud%1:19:01::","call_ids":["senseval3.d002.s056.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cloud%1:19:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.004370000000000001,"input_uncached_usd":0.0023000000000000004,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.434835124993697}
{"item_id":"senseval3.d002.s056.t002","gold_sense_keys":["bay%1:17:00::"],"candidates":[{"index":1,"sense_key":"bay%1:06:00::","synset_id":"alcove.n.01"},{"index":2,"sense_key":"bay%1:05:00::","synset_id":"bay.n.07"},{"index":3,"sense_key":"bay%1:11:00::","synset_id":"bay.n.02"},{"index":4,"sense_key":"bay%1:06:01::","synset_id":"bay.n.05"},{"index":5,"sense_key":"bay%1:06:02::","synset_id":"bay.n.04"},{"index":6,"sense_key":"bay%1:17:00::","synset_id":"bay.n.01"},{"index":7,"sense_key":"bay%1:20:00::","synset_id":"true_laurel.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"bay%1:17:00::","call_ids":["senseval3.d002.s056.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"bay%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.0045000000000000005,"input_uncached_usd":0.00249,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8771475829416886}
{"item_id":"senseval3.d002.s056.t004","gold_sense_keys":["fire%1:11:00::"],"candidates":[{"index":1,"sense_key":"fire%1:27:00::","synset_id":"fire.n.07"},{"index":2,"sense_key":"fire%1:11:01::","synset_id":"fire.n.08"},{"index":3,"sense_key":"fire%1:11:00::","synset_id":"fire.n.01"},{"index":4,"sense_key":"fire%1:04:00::","synset_id":"fire.n.02"},{"index":5,"sense_key":"fire%1:27:01::","synset_id":"fire.n.05"},{"index":6,"sense_key":"fire%1:10:00::","synset_id":"fire.n.09"},{"index":7,"sense_key":"fire%1:22:00::","synset_id":"fire.n.03"},{"index":8,"sense_key":"fire%1:06:00::","synset_id":"fire.n.04"},{"index":9,"sense_key":"fire%1:12:00::","synset_id":"ardor.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fire%1:11:00::","call_ids":["senseval3.d002.s056.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fire%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.006125,"input_uncached_usd":0.0031850000000000003,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.215005000005476}
{"item_id":"senseval3.d002.s057.t000","gold_sense_keys":["feel%2:39:00::"],"candidates":[{"index":1,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":2,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"},{"index":3,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":4,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":5,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":6,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":7,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":8,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":9,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":10,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":11,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":12,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":13,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"feel%2:39:00::","call_ids":["senseval3.d002.s057.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"feel%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":791,"cached_input_tokens":0,"output_tokens":159,"reasoning_output_tokens":143},"cost":{"currency":"USD","total_usd":0.008725,"input_uncached_usd":0.003955,"input_cached_usd":0.0,"output_usd":0.00477,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.676206332980655}
{"item_id":"senseval3.d002.s058.t000","gold_sense_keys":["back%1:08:00::"],"candidates":[{"index":1,"sense_key":"back%1:06:00::","synset_id":"rear.n.05"},{"index":2,"sense_key":"back%1:06:02::","synset_id":"back.n.07"},{"index":3,"sense_key":"back%1:04:00::","synset_id":"back.n.09"},{"index":4,"sense_key":"back%1:18:00::","synset_id":"back.n.04"},{"index":5,"sense_key":"back%1:15:02::","synset_id":"back.n.03"},{"index":6,"sense_key":"back%1:08:00::","synset_id":"back.n.01"},{"index":7,"sense_key":"back%1:06:03::","synset_id":"binding.n.05"},{"index":8,"sense_key":"back%1:06:01::","synset_id":"back.n.08"},{"index":9,"sense_key":"back%1:08:01::","synset_id":"spinal_column.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"back%1:08:00::","call_ids":["senseval3.d002.s058.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"back%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":86},"cost":{"currency":"USD","total_usd":0.006365000000000001,"input_uncached_usd":0.003305,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.43188079202082}
{"item_id":"senseval3.d002.s058.t002","gold_sense_keys":["hand%1:08:00::"],"candidates":[{"index":1,"sense_key":"hand%1:06:00::","synset_id":"hand.n.08"},{"index":2,"sense_key":"hand%1:18:00::","synset_id":"hired_hand.n.01"},{"index":3,"sense_key":"hand%1:04:00::","synset_id":"hand.n.14"},{"index":4,"sense_key":"hand%1:08:00::","synset_id":"hand.n.01"},{"index":5,"sense_key":"hand%1:05:01::","synset_id":"hand.n.13"},{"index":6,"sense_key":"hand%1:15:00::","synset_id":"hand.n.05"},{"index":7,"sense_key":"hand%1:09:01::","synset_id":"hand.n.07"},{"index":8,"sense_key":"hand%1:18:02::","synset_id":"hand.n.10"},{"index":9,"sense_key":"hand%1:14:00::","synset_id":"hand.n.06"},{"index":10,"sense_key":"hand%1:10:01::","synset_id":"hand.n.12"},{"index":11,"sense_key":"hand%1:23:00::","synset_id":"hand.n.09"},{"index":12,"sense_key":"hand%1:10:00::","synset_id":"handwriting.n.01"},{"index":13,"sense_key":"hand%1:09:00::","synset_id":"hand.n.04"},{"index":14,"sense_key":"hand%1:18:01::","synset_id":"bridge_player.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"hand%1:08:00::","call_ids":["senseval3.d002.s058.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"hand%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":842,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.00634,"input_uncached_usd":0.00421,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8964673340087757}
{"item_id":"senseval3.d002.s058.t003","gold_sense_keys":["shake%2:29:00::","shake%2:38:00::","shake%2:38:01::","shake%2:38:02::"],"candidates":[{"index":1,"sense_key":"shake%2:38:02::","synset_id":"rock.v.01"},{"index":2,"sense_key":"shake%2:38:01::","synset_id":"judder.v.01"},{"index":3,"sense_key":"shake%2:30:00::","synset_id":"shake.v.05"},{"index":4,"sense_key":"shake%2:30:01::","synset_id":"shake.v.08"},{"index":5,"sense_key":"shake%2:37:00::","synset_id":"stimulate.v.03"},{"index":6,"sense_key":"shake%2:32:00::","synset_id":"shake.v.09"},{"index":7,"sense_key":"shake%2:38:04::","synset_id":"shake.v.07"},{"index":8,"sense_key":"shake%2:38:00::","synset_id":"shake.v.01"},{"index":9,"sense_key":"shake%2:29:00::","synset_id":"shake.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"shake%2:29:00::","call_ids":["senseval3.d002.s058.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"shake%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":646,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01907,"input_uncached_usd":0.0032300000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.228737500030547}
{"item_id":"senseval3.d002.s059.t000","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:30:00::","synset_id":"think.v.13"},{"index":2,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":5,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":6,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":7,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":8,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":9,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":10,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":11,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":12,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":13,"sense_key":"think%2:31:14::","synset_id":"think.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval3.d002.s059.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":810,"cached_input_tokens":0,"output_tokens":370,"reasoning_output_tokens":354},"cost":{"currency":"USD","total_usd":0.01515,"input_uncached_usd":0.004050000000000001,"input_cached_usd":0.0,"output_usd":0.0111,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.3970546670025215}
{"item_id":"senseval3.d002.s059.t003","gold_sense_keys":["body%1:08:00::"],"candidates":[{"index":1,"sense_key":"body%1:14:00::","synset_id":"body.n.02"},{"index":2,"sense_key":"body%1:07:00::","synset_id":"consistency.n.01"},{"index":3,"sense_key":"body%1:08:01::","synset_id":"torso.n.01"},{"index":4,"sense_key":"body%1:06:01::","synset_id":"soundbox.n.01"},{"index":5,"sense_key":"body%1:08:00::","synset_id":"body.n.01"},{"index":6,"sense_key":"body%1:10:00::","synset_id":"body.n.08"},{"index":7,"sense_key":"body%1:17:00::","synset_id":"body.n.04"},{"index":8,"sense_key":"body%1:08:02::","synset_id":"body.n.03"},{"index":9,"sense_key":"body%1:07:01::","synset_id":"body.n.09"},{"index":10,"sense_key":"body%1:06:00::","synset_id":"body.n.11"},{"index":11,"sense_key":"body%1:14:02::","synset_id":"body.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"body%1:08:00::","call_ids":["senseval3.d002.s059.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"body%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.004895,"input_uncached_usd":0.003455,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4913113749353215}
{"item_id":"senseval3.d002.s059.t004","gold_sense_keys":["shake%2:29:00::","shake%2:38:00::","shake%2:38:01::","shake%2:38:02::"],"candidates":[{"index":1,"sense_key":"shake%2:38:01::","synset_id":"judder.v.01"},{"index":2,"sense_key":"shake%2:38:00::","synset_id":"shake.v.01"},{"index":3,"sense_key":"shake%2:30:01::","synset_id":"shake.v.08"},{"index":4,"sense_key":"shake%2:38:04::","synset_id":"shake.v.07"},{"index":5,"sense_key":"shake%2:32:00::","synset_id":"shake.v.09"},{"index":6,"sense_key":"shake%2:37:00::","synset_id":"stimulate.v.03"},{"index":7,"sense_key":"shake%2:38:02::","synset_id":"rock.v.01"},{"index":8,"sense_key":"shake%2:29:00::","synset_id":"shake.v.02"},{"index":9,"sense_key":"shake%2:30:00::","synset_id":"shake.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"shake%2:29:00::","call_ids":["senseval3.d002.s059.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"shake%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01901,"input_uncached_usd":0.00317,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.11417912505567}
{"item_id":"senseval3.d002.s062.t000","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":2,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":3,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":4,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":5,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":6,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":7,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":10,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":11,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":12,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":13,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":16,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":17,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":18,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":19,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":20,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":21,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":22,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":23,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":24,"sense_key":"see%2:39:11::","synset_id":"see.v.19"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval3.d002.s062.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1365,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.009165,"input_uncached_usd":0.006825,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2583704170538113}
{"item_id":"senseval3.d002.s062.t002","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":2,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":3,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":4,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":5,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":6,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":7,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":8,"sense_key":"house%1:06:01::","synset_id":"house.n.12"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:14:03::","synset_id":"house.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s062.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0053950000000000005,"input_uncached_usd":0.0034750000000000002,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8872447920730338}
{"item_id":"senseval3.d002.s062.t003","gold_sense_keys":["bay%1:17:00::"],"candidates":[{"index":1,"sense_key":"bay%1:06:02::","synset_id":"bay.n.04"},{"index":2,"sense_key":"bay%1:05:00::","synset_id":"bay.n.07"},{"index":3,"sense_key":"bay%1:06:00::","synset_id":"alcove.n.01"},{"index":4,"sense_key":"bay%1:11:00::","synset_id":"bay.n.02"},{"index":5,"sense_key":"bay%1:06:01::","synset_id":"bay.n.05"},{"index":6,"sense_key":"bay%1:17:00::","synset_id":"bay.n.01"},{"index":7,"sense_key":"bay%1:20:00::","synset_id":"true_laurel.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"bay%1:17:00::","call_ids":["senseval3.d002.s062.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"bay%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00429,"input_uncached_usd":0.00237,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.871842125081457}
{"item_id":"senseval3.d002.s063.t000","gold_sense_keys":["hard%3:00:06::"],"candidates":[{"index":1,"sense_key":"hard%5:00:00:alcoholic:00","synset_id":"hard.s.04"},{"index":2,"sense_key":"hard%5:00:00:stale:00","synset_id":"hard.s.06"},{"index":3,"sense_key":"hard%3:00:02::","synset_id":"hard.a.02"},{"index":4,"sense_key":"hard%3:00:05::","synset_id":"hard.a.07"},{"index":5,"sense_key":"hard%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":6,"sense_key":"hard%5:00:01:effortful:00","synset_id":"arduous.s.01"},{"index":7,"sense_key":"hard%3:00:01::","synset_id":"hard.a.03"},{"index":8,"sense_key":"hard%5:00:00:bad:00","synset_id":"hard.s.05"},{"index":9,"sense_key":"hard%3:00:00::","synset_id":"unvoiced.a.01"},{"index":10,"sense_key":"hard%3:00:03::","synset_id":"hard.a.08"},{"index":11,"sense_key":"hard%3:00:06::","synset_id":"difficult.a.01"},{"index":12,"sense_key":"hard%5:00:00:indulgent:00","synset_id":"intemperate.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"hard%3:00:06::","call_ids":["senseval3.d002.s063.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"hard%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":748,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00557,"input_uncached_usd":0.0037400000000000003,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4844027500366792}
{"item_id":"senseval3.d002.s063.t001","gold_sense_keys":["believe%2:31:00::"],"candidates":[{"index":1,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":2,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"},{"index":3,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"},{"index":4,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"},{"index":5,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"believe%2:31:00::","call_ids":["senseval3.d002.s063.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"believe%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.0046,"input_uncached_usd":0.0022,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9316910420311615}
{"item_id":"senseval3.d002.s063.t002","gold_sense_keys":["happen%2:30:00::"],"candidates":[{"index":1,"sense_key":"happen%2:41:00::","synset_id":"happen.v.03"},{"index":2,"sense_key":"happen%2:30:00::","synset_id":"happen.v.01"},{"index":3,"sense_key":"happen%2:30:01::","synset_id":"happen.v.04"},{"index":4,"sense_key":"happen%2:30:02::","synset_id":"happen.v.02"},{"index":5,"sense_key":"happen%2:40:12::","synset_id":"find.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"happen%2:30:00::","call_ids":["senseval3.d002.s063.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"happen%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":96},"cost":{"currency":"USD","total_usd":0.005730000000000001,"input_uncached_usd":0.00237,"input_cached_usd":0.0,"output_usd":0.00336,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.238291542045772}
{"item_id":"senseval3.d002.s066.t000","gold_sense_keys":["building%1:06:00::"],"candidates":[{"index":1,"sense_key":"building%1:04:01::","synset_id":"construction.n.07"},{"index":2,"sense_key":"building%1:04:00::","synset_id":"construction.n.01"},{"index":3,"sense_key":"building%1:06:00::","synset_id":"building.n.01"},{"index":4,"sense_key":"building%1:14:00::","synset_id":"building.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"building%1:06:00::","call_ids":["senseval3.d002.s066.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"building%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":397,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.0033950000000000004,"input_uncached_usd":0.0019850000000000002,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4087318750098348}
{"item_id":"senseval3.d002.s066.t001","gold_sense_keys":["corner%1:06:00::"],"candidates":[{"index":1,"sense_key":"corner%1:25:00::","synset_id":"recess.n.02"},{"index":2,"sense_key":"corner%1:15:03::","synset_id":"corner.n.10"},{"index":3,"sense_key":"corner%1:06:00::","synset_id":"corner.n.04"},{"index":4,"sense_key":"corner%1:06:01::","synset_id":"corner.n.11"},{"index":5,"sense_key":"corner%1:06:02::","synset_id":"corner.n.03"},{"index":6,"sense_key":"corner%1:15:01::","synset_id":"corner.n.05"},{"index":7,"sense_key":"corner%1:26:01::","synset_id":"corner.n.08"},{"index":8,"sense_key":"corner%1:15:00::","synset_id":"corner.n.02"},{"index":9,"sense_key":"corner%1:26:00::","synset_id":"corner.n.07"},{"index":10,"sense_key":"corner%1:15:02::","synset_id":"corner.n.01"},{"index":11,"sense_key":"corner%1:17:00::","synset_id":"corner.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"corner%1:06:00::","call_ids":["senseval3.d002.s066.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"corner%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.00534,"input_uncached_usd":0.0030600000000000002,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5511127080535516}
{"item_id":"senseval3.d002.s066.t002","gold_sense_keys":["damage%2:30:00::"],"candidates":[{"index":1,"sense_key":"damage%2:30:00::","synset_id":"damage.v.01"},{"index":2,"sense_key":"damage%2:30:01::","synset_id":"damage.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%2:30:00::","call_ids":["senseval3.d002.s066.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":284,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":103},"cost":{"currency":"USD","total_usd":0.00496,"input_uncached_usd":0.00142,"input_cached_usd":0.0,"output_usd":0.00354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.751427458017133}
{"item_id":"senseval3.d002.s066.t003","gold_sense_keys":["old%3:00:02::"],"candidates":[{"index":1,"sense_key":"old%5:00:03:past:00","synset_id":"erstwhile.s.01"},{"index":2,"sense_key":"old%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"},{"index":3,"sense_key":"old%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":4,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"},{"index":5,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"},{"index":6,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":7,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":8,"sense_key":"old%5:00:00:preceding:00","synset_id":"previous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"old%3:00:02::","call_ids":["senseval3.d002.s066.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"old%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004945,"input_uncached_usd":0.002935,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.589911957969889}
{"item_id":"senseval3.d002.s066.t004","gold_sense_keys":["lady%1:18:02::"],"candidates":[{"index":1,"sense_key":"lady%1:18:00::","synset_id":"lady.n.03"},{"index":2,"sense_key":"lady%1:18:02::","synset_id":"lady.n.01"},{"index":3,"sense_key":"lady%1:18:01::","synset_id":"dame.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"lady%1:18:02::","call_ids":["senseval3.d002.s066.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"lady%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":324,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.00348,"input_uncached_usd":0.0016200000000000001,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6406627499964088}
{"item_id":"senseval3.d002.s066.t005","gold_sense_keys":["old%3:00:02::"],"candidates":[{"index":1,"sense_key":"old%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"},{"index":2,"sense_key":"old%5:00:00:preceding:00","synset_id":"previous.s.01"},{"index":3,"sense_key":"old%5:00:03:past:00","synset_id":"erstwhile.s.01"},{"index":4,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":5,"sense_key":"old%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":6,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"},{"index":7,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":8,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"old%3:00:02::","call_ids":["senseval3.d002.s066.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"old%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.005005000000000001,"input_uncached_usd":0.002935,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.242951584048569}
{"item_id":"senseval3.d002.s066.t006","gold_sense_keys":["mother%1:18:00::"],"candidates":[{"index":1,"sense_key":"mother%1:18:00::","synset_id":"mother.n.01"},{"index":2,"sense_key":"mother%1:09:00::","synset_id":"mother.n.05"},{"index":3,"sense_key":"mother%1:27:00::","synset_id":"mother.n.02"},{"index":4,"sense_key":"mother%1:18:01::","synset_id":"mother.n.04"},{"index":5,"sense_key":"mother%1:18:02::","synset_id":"mother.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mother%1:18:00::","call_ids":["senseval3.d002.s066.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mother%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.00381,"input_uncached_usd":0.0019500000000000001,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5881485419813544}
{"item_id":"senseval3.d002.s067.t000","gold_sense_keys":["book%1:06:00::"],"candidates":[{"index":1,"sense_key":"book%1:10:04::","synset_id":"bible.n.01"},{"index":2,"sense_key":"book%1:14:01::","synset_id":"book.n.06"},{"index":3,"sense_key":"book%1:10:01::","synset_id":"script.n.01"},{"index":4,"sense_key":"book%1:10:00::","synset_id":"book.n.01"},{"index":5,"sense_key":"book%1:10:02::","synset_id":"record.n.05"},{"index":6,"sense_key":"book%1:10:05::","synset_id":"koran.n.01"},{"index":7,"sense_key":"book%1:06:00::","synset_id":"book.n.02"},{"index":8,"sense_key":"book%1:06:02::","synset_id":"book.n.11"},{"index":9,"sense_key":"book%1:10:03::","synset_id":"book.n.10"},{"index":10,"sense_key":"book%1:14:00::","synset_id":"book.n.07"},{"index":11,"sense_key":"book%1:21:00::","synset_id":"ledger.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"book%1:06:00::","call_ids":["senseval3.d002.s067.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"book%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":112},"cost":{"currency":"USD","total_usd":0.007330000000000001,"input_uncached_usd":0.0034900000000000005,"input_cached_usd":0.0,"output_usd":0.00384,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.708093125023879}
{"item_id":"senseval3.d002.s068.t000","gold_sense_keys":["type%2:32:00::"],"candidates":[{"index":1,"sense_key":"type%2:32:00::","synset_id":"type.v.01"},{"index":2,"sense_key":"type%2:31:00::","synset_id":"type.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%2:32:00::","call_ids":["senseval3.d002.s068.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":285,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.003075,"input_uncached_usd":0.001425,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.659541541012004}
{"item_id":"senseval3.d002.s068.t001","gold_sense_keys":["standing%3:00:00::"],"candidates":[{"index":1,"sense_key":"standing%5:00:00:permanent:00","synset_id":"standing.s.02"},{"index":2,"sense_key":"standing%3:00:01::","synset_id":"standing.a.04"},{"index":3,"sense_key":"standing%5:00:00:erect:00","synset_id":"standing.s.01"},{"index":4,"sense_key":"standing%3:00:00::","synset_id":"standing.a.05"},{"index":5,"sense_key":"standing%5:00:00:regular:02","synset_id":"standing.s.03"},{"index":6,"sense_key":"standing%3:00:02::","synset_id":"standing.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"standing%3:00:00::","call_ids":["senseval3.d002.s068.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"standing%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.005560000000000001,"input_uncached_usd":0.00205,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4172497499966994}
{"item_id":"senseval3.d002.s068.t002","gold_sense_keys":["position%1:07:01::"],"candidates":[{"index":1,"sense_key":"position%1:10:01::","synset_id":"place.n.15"},{"index":2,"sense_key":"position%1:15:00::","synset_id":"position.n.01"},{"index":3,"sense_key":"position%1:15:03::","synset_id":"position.n.08"},{"index":4,"sense_key":"position%1:04:00::","synset_id":"placement.n.03"},{"index":5,"sense_key":"position%1:26:02::","synset_id":"situation.n.02"},{"index":6,"sense_key":"position%1:09:01::","synset_id":"position.n.12"},{"index":7,"sense_key":"position%1:10:00::","synset_id":"side.n.10"},{"index":8,"sense_key":"position%1:15:02::","synset_id":"military_position.n.01"},{"index":9,"sense_key":"position%1:26:00::","synset_id":"status.n.01"},{"index":10,"sense_key":"position%1:09:00::","synset_id":"position.n.03"},{"index":11,"sense_key":"position%1:07:00::","synset_id":"position.n.07"},{"index":12,"sense_key":"position%1:04:01::","synset_id":"position.n.06"},{"index":13,"sense_key":"position%1:04:04::","synset_id":"position.n.16"},{"index":14,"sense_key":"position%1:04:02::","synset_id":"position.n.09"},{"index":15,"sense_key":"position%1:07:01::","synset_id":"position.n.04"},{"index":16,"sense_key":"position%1:04:03::","synset_id":"stead.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"position%1:07:01::","call_ids":["senseval3.d002.s068.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"position%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":877,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.006125,"input_uncached_usd":0.004385,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.398393707931973}
{"item_id":"senseval3.d002.s072.t000","gold_sense_keys":["feel%2:39:00::"],"candidates":[{"index":1,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"},{"index":2,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":3,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":4,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":5,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":6,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":7,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":8,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":9,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":10,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":11,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":12,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":13,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"feel%2:39:00::","call_ids":["senseval3.d002.s072.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"feel%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.006955,"input_uncached_usd":0.0037150000000000004,"input_cached_usd":0.0,"output_usd":0.0032400000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5660897080088034}
{"item_id":"senseval3.d002.s072.t001","gold_sense_keys":["thing%1:06:01::"],"candidates":[{"index":1,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":2,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":3,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":4,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":5,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":6,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":9,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":10,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"},{"index":11,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":12,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"thing%1:06:01::","call_ids":["senseval3.d002.s072.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"thing%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":1313,"reasoning_output_tokens":1285},"cost":{"currency":"USD","total_usd":0.04236,"input_uncached_usd":0.0029700000000000004,"input_cached_usd":0.0,"output_usd":0.03939,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":23.64645920903422}
{"item_id":"senseval3.d002.s072.t002","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"},{"index":2,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":4,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d002.s072.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":399,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":124},"cost":{"currency":"USD","total_usd":0.006195000000000001,"input_uncached_usd":0.0019950000000000002,"input_cached_usd":0.0,"output_usd":0.0042,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.875292459037155}
{"item_id":"senseval3.d002.s072.t003","gold_sense_keys":["strange%3:00:00::"],"candidates":[{"index":1,"sense_key":"strange%3:00:01::","synset_id":"foreign.a.02"},{"index":2,"sense_key":"strange%3:00:00::","synset_id":"strange.a.01"},{"index":3,"sense_key":"strange%5:00:01:unfamiliar:00","synset_id":"strange.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"strange%3:00:00::","call_ids":["senseval3.d002.s072.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"strange%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":344,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00367,"input_uncached_usd":0.0017200000000000002,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7827529999194667}
{"item_id":"senseval3.d002.s072.t004","gold_sense_keys":["bird%1:05:00::"],"candidates":[{"index":1,"sense_key":"bird%1:10:00::","synset_id":"boo.n.01"},{"index":2,"sense_key":"bird%1:18:00::","synset_id":"dame.n.01"},{"index":3,"sense_key":"bird%1:06:01::","synset_id":"shuttlecock.n.01"},{"index":4,"sense_key":"bird%1:13:00::","synset_id":"bird.n.02"},{"index":5,"sense_key":"bird%1:05:00::","synset_id":"bird.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"bird%1:05:00::","call_ids":["senseval3.d002.s072.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"bird%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":398,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.00346,"input_uncached_usd":0.00199,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.607410458032973}
{"item_id":"senseval3.d002.s072.t005","gold_sense_keys":["behavior%1:04:01::"],"candidates":[{"index":1,"sense_key":"behavior%1:04:00::","synset_id":"behavior.n.01"},{"index":2,"sense_key":"behavior%1:04:01::","synset_id":"behavior.n.04"},{"index":3,"sense_key":"behavior%1:07:00::","synset_id":"demeanor.n.01"},{"index":4,"sense_key":"behavior%1:26:00::","synset_id":"behavior.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"behavior%1:04:01::","call_ids":["senseval3.d002.s072.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"behavior%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":366,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.00507,"input_uncached_usd":0.0018300000000000002,"input_cached_usd":0.0,"output_usd":0.0032400000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9996822921093553}
{"item_id":"senseval3.d002.s073.t000","gold_sense_keys":["duck%1:05:00::"],"candidates":[{"index":1,"sense_key":"duck%1:23:00::","synset_id":"duck.n.02"},{"index":2,"sense_key":"duck%1:05:00::","synset_id":"duck.n.01"},{"index":3,"sense_key":"duck%1:13:00::","synset_id":"duck.n.03"},{"index":4,"sense_key":"duck%1:06:00::","synset_id":"duck.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"duck%1:05:00::","call_ids":["senseval3.d002.s073.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"duck%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":327,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.003945000000000001,"input_uncached_usd":0.0016350000000000002,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6664824170293286}
{"item_id":"senseval3.d002.s073.t001","gold_sense_keys":["swarm%1:14:00::"],"candidates":[{"index":1,"sense_key":"swarm%1:14:01::","synset_id":"drove.n.02"},{"index":2,"sense_key":"swarm%1:14:00::","synset_id":"swarm.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"swarm%1:14:00::","call_ids":["senseval3.d002.s073.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"swarm%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":274,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01721,"input_uncached_usd":0.0013700000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.73208095796872}
{"item_id":"senseval3.d002.s076.t000","gold_sense_keys":["feel%2:39:00::"],"candidates":[{"index":1,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":2,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"},{"index":3,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":4,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":5,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":6,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":7,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":8,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":9,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":10,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":11,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":12,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":13,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"feel%2:39:00::","call_ids":["senseval3.d002.s076.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"feel%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":745,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":111},"cost":{"currency":"USD","total_usd":0.007535,"input_uncached_usd":0.0037250000000000004,"input_cached_usd":0.0,"output_usd":0.00381,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0236638330388814}
{"item_id":"senseval3.d002.s076.t003","gold_sense_keys":["second%1:28:00::"],"candidates":[{"index":1,"sense_key":"second%1:18:00::","synset_id":"second.n.07"},{"index":2,"sense_key":"second%1:28:00::","synset_id":"second.n.01"},{"index":3,"sense_key":"second%1:06:01::","synset_id":"irregular.n.02"},{"index":4,"sense_key":"second%1:06:00::","synset_id":"second_gear.n.01"},{"index":5,"sense_key":"second%1:10:00::","synset_id":"second.n.08"},{"index":6,"sense_key":"second%1:24:00::","synset_id":"second.n.05"},{"index":7,"sense_key":"second%1:28:01::","synset_id":"moment.n.02"},{"index":8,"sense_key":"second%1:23:00::","synset_id":"second.n.06"},{"index":9,"sense_key":"second%1:04:00::","synset_id":"second_base.n.02"},{"index":10,"sense_key":"second%1:28:02::","synset_id":"moment.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"second%1:28:00::","call_ids":["senseval3.d002.s076.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"second%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":137},"cost":{"currency":"USD","total_usd":0.007665,"input_uncached_usd":0.003075,"input_cached_usd":0.0,"output_usd":0.00459,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0202522919280455}
{"item_id":"senseval3.d002.s077.t000","gold_sense_keys":["numb%5:00:00:afraid:00"],"candidates":[{"index":1,"sense_key":"numb%5:00:00:afraid:00","synset_id":"numb.s.03"},{"index":2,"sense_key":"numb%5:00:00:insensitive:02","synset_id":"dead.s.04"},{"index":3,"sense_key":"numb%5:00:00:insensible:00","synset_id":"asleep.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"numb%5:00:00:afraid:00","call_ids":["senseval3.d002.s077.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"numb%5:00:00:afraid:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":347,"cached_input_tokens":0,"output_tokens":328,"reasoning_output_tokens":312},"cost":{"currency":"USD","total_usd":0.011575,"input_uncached_usd":0.0017350000000000002,"input_cached_usd":0.0,"output_usd":0.00984,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.441863083047792}
{"item_id":"senseval3.d002.s080.t001","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":2,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval3.d002.s080.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"seem%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":408,"cached_input_tokens":0,"output_tokens":518,"reasoning_output_tokens":502},"cost":{"currency":"USD","total_usd":0.01758,"input_uncached_usd":0.00204,"input_cached_usd":0.0,"output_usd":0.01554,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.112696124939248}
{"item_id":"senseval3.d002.s080.t002","gold_sense_keys":["part%1:09:00::","part%1:15:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":2,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":3,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":4,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":5,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":6,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":7,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":8,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":9,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":10,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":11,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"part%1:15:00::","call_ids":["senseval3.d002.s080.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"part%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":817,"cached_input_tokens":0,"output_tokens":489,"reasoning_output_tokens":473},"cost":{"currency":"USD","total_usd":0.018755,"input_uncached_usd":0.004085,"input_cached_usd":0.0,"output_usd":0.01467,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.589354875031859}
{"item_id":"senseval3.d002.s080.t003","gold_sense_keys":["town%1:15:00::"],"candidates":[{"index":1,"sense_key":"town%1:18:00::","synset_id":"town.n.04"},{"index":2,"sense_key":"town%1:14:00::","synset_id":"town.n.02"},{"index":3,"sense_key":"town%1:15:00::","synset_id":"town.n.01"},{"index":4,"sense_key":"town%1:15:01::","synset_id":"township.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"town%1:15:00::","call_ids":["senseval3.d002.s080.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"town%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.00459,"input_uncached_usd":0.00198,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9431975840125233}
{"item_id":"senseval3.d002.s081.t001","gold_sense_keys":["phone%1:06:00::"],"candidates":[{"index":1,"sense_key":"phone%1:10:00::","synset_id":"phone.n.02"},{"index":2,"sense_key":"phone%1:06:01::","synset_id":"earphone.n.01"},{"index":3,"sense_key":"phone%1:06:00::","synset_id":"telephone.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"phone%1:06:00::","call_ids":["senseval3.d002.s081.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"phone%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.0040750000000000005,"input_uncached_usd":0.002095,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7505081250565127}
{"item_id":"senseval3.d002.s081.t002","gold_sense_keys":["mess%1:23:00::","mess%1:26:00::"],"candidates":[{"index":1,"sense_key":"mess%1:26:00::","synset_id":"mess.n.01"},{"index":2,"sense_key":"mess%1:26:02::","synset_id":"fix.n.01"},{"index":3,"sense_key":"mess%1:13:01::","synset_id":"mess.n.03"},{"index":4,"sense_key":"mess%1:23:00::","synset_id":"batch.n.02"},{"index":5,"sense_key":"mess%1:13:00::","synset_id":"mess.n.04"},{"index":6,"sense_key":"mess%1:06:00::","synset_id":"mess.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mess%1:26:00::","call_ids":["senseval3.d002.s081.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mess%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":492,"reasoning_output_tokens":476},"cost":{"currency":"USD","total_usd":0.01729,"input_uncached_usd":0.00253,"input_cached_usd":0.0,"output_usd":0.01476,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":15.372331666992977}
{"item_id":"senseval3.d002.s081.t004","gold_sense_keys":["wine%1:13:00::"],"candidates":[{"index":1,"sense_key":"wine%1:13:00::","synset_id":"wine.n.01"},{"index":2,"sense_key":"wine%1:07:00::","synset_id":"wine.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wine%1:13:00::","call_ids":["senseval3.d002.s081.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wine%1:13:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":310,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.00353,"input_uncached_usd":0.0015500000000000002,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.99413858295884}
{"item_id":"senseval3.d002.s081.t005","gold_sense_keys":["floor%1:06:00::"],"candidates":[{"index":1,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"},{"index":2,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"},{"index":3,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":4,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":5,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"},{"index":6,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"},{"index":7,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"},{"index":8,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":9,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":10,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"floor%1:06:00::","call_ids":["senseval3.d002.s081.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"floor%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.005345,"input_uncached_usd":0.003245,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7719568749889731}
{"item_id":"senseval3.d002.s081.t006","gold_sense_keys":["big%3:00:01::"],"candidates":[{"index":1,"sense_key":"big%5:00:02:proud:00","synset_id":"boastful.s.01"},{"index":2,"sense_key":"big%5:00:00:conspicuous:00","synset_id":"big.s.04"},{"index":3,"sense_key":"big%5:00:00:heavy:08","synset_id":"big.s.09"},{"index":4,"sense_key":"big%3:00:01::","synset_id":"large.a.01"},{"index":5,"sense_key":"big%5:00:00:intemperate:00","synset_id":"big.s.05"},{"index":6,"sense_key":"big%5:00:00:generous:01","synset_id":"big.s.11"},{"index":7,"sense_key":"big%5:00:00:pregnant:00","synset_id":"big.s.12"},{"index":8,"sense_key":"big%5:00:00:generous:02","synset_id":"big.s.10"},{"index":9,"sense_key":"big%5:00:00:mature:01","synset_id":"adult.s.01"},{"index":10,"sense_key":"big%5:00:00:intense:00","synset_id":"bad.s.01"},{"index":11,"sense_key":"big%5:00:00:important:00","synset_id":"big.s.01"},{"index":12,"sense_key":"big%5:00:00:proud:00","synset_id":"big.s.07"},{"index":13,"sense_key":"big%5:00:00:loud:00","synset_id":"big.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"big%3:00:01::","call_ids":["senseval3.d002.s081.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"big%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":841,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.0057350000000000005,"input_uncached_usd":0.004205,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.492756292107515}
{"item_id":"senseval3.d002.s081.t007","gold_sense_keys":["old%3:00:01::"],"candidates":[{"index":1,"sense_key":"old%5:00:03:past:00","synset_id":"erstwhile.s.01"},{"index":2,"sense_key":"old%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":3,"sense_key":"old%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"},{"index":4,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":5,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"},{"index":6,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":7,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"},{"index":8,"sense_key":"old%5:00:00:preceding:00","synset_id":"previous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"old%3:00:01::","call_ids":["senseval3.d002.s081.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"old%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018930000000000002,"input_uncached_usd":0.0031200000000000004,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.763530792086385}
{"item_id":"senseval3.d002.s082.t000","gold_sense_keys":["quiver%1:26:00::"],"candidates":[{"index":1,"sense_key":"quiver%1:04:00::","synset_id":"vibration.n.01"},{"index":2,"sense_key":"quiver%1:06:00::","synset_id":"quiver.n.03"},{"index":3,"sense_key":"quiver%1:12:00::","synset_id":"frisson.n.01"},{"index":4,"sense_key":"quiver%1:26:00::","synset_id":"shaking.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"quiver%1:26:00::","call_ids":["senseval3.d002.s082.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"quiver%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":111},"cost":{"currency":"USD","total_usd":0.005860000000000001,"input_uncached_usd":0.00205,"input_cached_usd":0.0,"output_usd":0.00381,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.726676999940537}
{"item_id":"senseval3.d002.s082.t002","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":3,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":4,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":5,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":6,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":7,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":8,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":9,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":10,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":11,"sense_key":"house%1:06:01::","synset_id":"house.n.12"},{"index":12,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s082.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.0056500000000000005,"input_uncached_usd":0.0036400000000000004,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5902362500783056}
{"item_id":"senseval3.d002.s082.t004","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":2,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":3,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":4,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"},{"index":5,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":6,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"minute%1:28:00::","call_ids":["senseval3.d002.s082.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":136},"cost":{"currency":"USD","total_usd":0.007050000000000001,"input_uncached_usd":0.00249,"input_cached_usd":0.0,"output_usd":0.00456,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.897246041917242}
{"item_id":"senseval3.d002.s082.t005","gold_sense_keys":["unpredictable%5:00:00:sporadic:00"],"candidates":[{"index":1,"sense_key":"unpredictable%5:00:00:sporadic:00","synset_id":"irregular.s.01"},{"index":2,"sense_key":"unpredictable%3:00:00::","synset_id":"unpredictable.a.01"},{"index":3,"sense_key":"unpredictable%5:00:00:indeterminable:00","synset_id":"unpredictable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"unpredictable%5:00:00:sporadic:00","call_ids":["senseval3.d002.s082.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"unpredictable%5:00:00:sporadic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":353,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":109},"cost":{"currency":"USD","total_usd":0.005515,"input_uncached_usd":0.001765,"input_cached_usd":0.0,"output_usd":0.0037500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.384048750041984}
{"item_id":"senseval3.d002.s082.t006","gold_sense_keys":["interval%1:28:00::"],"candidates":[{"index":1,"sense_key":"interval%1:10:00::","synset_id":"interval.n.04"},{"index":2,"sense_key":"interval%1:07:00::","synset_id":"interval.n.03"},{"index":3,"sense_key":"interval%1:28:00::","synset_id":"time_interval.n.01"},{"index":4,"sense_key":"interval%1:09:00::","synset_id":"interval.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"interval%1:28:00::","call_ids":["senseval3.d002.s082.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"interval%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":382,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.0037400000000000003,"input_uncached_usd":0.0019100000000000002,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7452899999916553}
{"item_id":"senseval3.d002.s082.t007","gold_sense_keys":["mouse%1:05:00::"],"candidates":[{"index":1,"sense_key":"mouse%1:06:00::","synset_id":"mouse.n.04"},{"index":2,"sense_key":"mouse%1:18:00::","synset_id":"mouse.n.03"},{"index":3,"sense_key":"mouse%1:26:00::","synset_id":"shiner.n.01"},{"index":4,"sense_key":"mouse%1:05:00::","synset_id":"mouse.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"mouse%1:05:00::","call_ids":["senseval3.d002.s082.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"mouse%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":430,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.00371,"input_uncached_usd":0.00215,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.31533320795279}
{"item_id":"senseval3.d002.s082.t008","gold_sense_keys":["live%2:42:08::"],"candidates":[{"index":1,"sense_key":"live%2:42:08::","synset_id":"populate.v.01"},{"index":2,"sense_key":"live%2:42:07::","synset_id":"live.v.07"},{"index":3,"sense_key":"live%2:42:04::","synset_id":"survive.v.01"},{"index":4,"sense_key":"live%2:42:06::","synset_id":"live.v.02"},{"index":5,"sense_key":"live%2:42:00::","synset_id":"be.v.11"},{"index":6,"sense_key":"live%2:42:01::","synset_id":"exist.v.02"},{"index":7,"sense_key":"live%2:31:00::","synset_id":"know.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"live%2:42:08::","call_ids":["senseval3.d002.s082.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"live%2:42:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.004945000000000001,"input_uncached_usd":0.0030250000000000003,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8251069170655683}
{"item_id":"senseval3.d002.s082.t010","gold_sense_keys":["take%2:40:08::"],"candidates":[{"index":1,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":2,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":3,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":4,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":5,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":6,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":7,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":8,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":9,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":10,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":11,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":12,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":13,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":14,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":15,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":16,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":17,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":18,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":19,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":20,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"},{"index":21,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":22,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":23,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":24,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":25,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":26,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":27,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":28,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":29,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":30,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":31,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":32,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":33,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":34,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":35,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":36,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":37,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":38,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":39,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":40,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:38:02::","synset_id":"take.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":18,"chosen_sense_key":"take%2:40:08::","call_ids":["senseval3.d002.s082.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":18,"predicted_sense_key":"take%2:40:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2117,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":96},"cost":{"currency":"USD","total_usd":0.013945,"input_uncached_usd":0.010585,"input_cached_usd":0.0,"output_usd":0.00336,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9393460829742253}
{"item_id":"senseval3.d002.s082.t011","gold_sense_keys":["refuge%1:06:00::"],"candidates":[{"index":1,"sense_key":"refuge%1:04:00::","synset_id":"recourse.n.01"},{"index":2,"sense_key":"refuge%1:07:00::","synset_id":"recourse.n.02"},{"index":3,"sense_key":"refuge%1:06:00::","synset_id":"refuge.n.03"},{"index":4,"sense_key":"refuge%1:15:00::","synset_id":"safety.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"refuge%1:06:00::","call_ids":["senseval3.d002.s082.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"refuge%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01792,"input_uncached_usd":0.0020800000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.667596625047736}
{"item_id":"senseval3.d002.s083.t001","gold_sense_keys":["distressed%5:00:01:troubled:00","distressed%5:00:02:troubled:00"],"candidates":[{"index":1,"sense_key":"distressed%5:00:00:troubled:00","synset_id":"distressed.s.01"},{"index":2,"sense_key":"distressed%3:00:04::","synset_id":"dysphoric.a.01"},{"index":3,"sense_key":"distressed%5:00:02:troubled:00","synset_id":"stressed.s.01"},{"index":4,"sense_key":"distressed%5:00:01:troubled:00","synset_id":"disquieted.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"distressed%5:00:01:troubled:00","call_ids":["senseval3.d002.s083.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"distressed%5:00:01:troubled:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":88},"cost":{"currency":"USD","total_usd":0.005485,"input_uncached_usd":0.0023650000000000003,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7517895420314744}
{"item_id":"senseval3.d002.s084.t001","gold_sense_keys":["roll%2:38:01::","roll%2:38:10::"],"candidates":[{"index":1,"sense_key":"roll%2:30:06::","synset_id":"roll.v.17"},{"index":2,"sense_key":"roll%2:38:12::","synset_id":"roll.v.13"},{"index":3,"sense_key":"roll%2:35:05::","synset_id":"roll_out.v.01"},{"index":4,"sense_key":"roll%2:40:00::","synset_id":"hustle.v.03"},{"index":5,"sense_key":"roll%2:35:01::","synset_id":"roll.v.07"},{"index":6,"sense_key":"roll%2:30:08::","synset_id":"seethe.v.04"},{"index":7,"sense_key":"roll%2:30:10::","synset_id":"roll.v.08"},{"index":8,"sense_key":"roll%2:38:00::","synset_id":"roll.v.01"},{"index":9,"sense_key":"roll%2:39:00::","synset_id":"roll.v.05"},{"index":10,"sense_key":"roll%2:38:10::","synset_id":"roll.v.11"},{"index":11,"sense_key":"roll%2:38:01::","synset_id":"wheel.v.03"},{"index":12,"sense_key":"roll%2:32:00::","synset_id":"roll.v.15"},{"index":13,"sense_key":"roll%2:38:09::","synset_id":"roll.v.12"},{"index":14,"sense_key":"roll%2:42:00::","synset_id":"roll.v.03"},{"index":15,"sense_key":"roll%2:29:06::","synset_id":"roll.v.09"},{"index":16,"sense_key":"roll%2:30:07::","synset_id":"roll.v.18"},{"index":17,"sense_key":"roll%2:38:11::","synset_id":"roll.v.14"},{"index":18,"sense_key":"roll%2:35:00::","synset_id":"wind.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"roll%2:38:10::","call_ids":["senseval3.d002.s084.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"roll%2:38:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1006,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02087,"input_uncached_usd":0.005030000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.007968583027832}
{"item_id":"senseval3.d002.s084.t002","gold_sense_keys":["town%1:15:00::"],"candidates":[{"index":1,"sense_key":"town%1:18:00::","synset_id":"town.n.04"},{"index":2,"sense_key":"town%1:15:01::","synset_id":"township.n.01"},{"index":3,"sense_key":"town%1:14:00::","synset_id":"town.n.02"},{"index":4,"sense_key":"town%1:15:00::","synset_id":"town.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"town%1:15:00::","call_ids":["senseval3.d002.s084.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"town%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.004605000000000001,"input_uncached_usd":0.0021750000000000003,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5979250420350581}
{"item_id":"senseval3.d002.s086.t001","gold_sense_keys":["get%2:30:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":2,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":3,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":4,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":5,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":8,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":9,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":10,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":11,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":12,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":13,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":14,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":15,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":16,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":17,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"},{"index":18,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":19,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":20,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":21,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":22,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":23,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":24,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":25,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":26,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":27,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":28,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":29,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":30,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":31,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":32,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":33,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":34,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":35,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":36,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":21,"chosen_sense_key":"get%2:30:00::","call_ids":["senseval3.d002.s086.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":21,"predicted_sense_key":"get%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1936,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.012230000000000001,"input_uncached_usd":0.009680000000000001,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8904439159668982}
{"item_id":"senseval3.d002.s086.t002","gold_sense_keys":["run%2:38:00::"],"candidates":[{"index":1,"sense_key":"run%2:38:05::","synset_id":"run.v.29"},{"index":2,"sense_key":"run%2:42:05::","synset_id":"run.v.30"},{"index":3,"sense_key":"run%2:30:04::","synset_id":"run.v.41"},{"index":4,"sense_key":"run%2:30:02::","synset_id":"move.v.13"},{"index":5,"sense_key":"run%2:42:03::","synset_id":"prevail.v.03"},{"index":6,"sense_key":"run%2:33:01::","synset_id":"race.v.02"},{"index":7,"sense_key":"run%2:42:04::","synset_id":"run.v.05"},{"index":8,"sense_key":"run%2:38:13::","synset_id":"run.v.34"},{"index":9,"sense_key":"run%2:38:04::","synset_id":"scat.v.01"},{"index":10,"sense_key":"run%2:42:06::","synset_id":"run.v.18"},{"index":11,"sense_key":"run%2:36:02::","synset_id":"carry.v.15"},{"index":12,"sense_key":"run%2:33:02::","synset_id":"hunt.v.01"},{"index":13,"sense_key":"run%2:41:04::","synset_id":"run.v.31"},{"index":14,"sense_key":"run%2:30:03::","synset_id":"run.v.14"},{"index":15,"sense_key":"run%2:42:00::","synset_id":"run.v.03"},{"index":16,"sense_key":"run%2:36:00::","synset_id":"run.v.21"},{"index":17,"sense_key":"run%2:38:00::","synset_id":"run.v.01"},{"index":18,"sense_key":"run%2:38:09::","synset_id":"ply.v.03"},{"index":19,"sense_key":"run%2:38:08::","synset_id":"run.v.27"},{"index":20,"sense_key":"run%2:38:03::","synset_id":"run.v.28"},{"index":21,"sense_key":"run%2:30:07::","synset_id":"run.v.15"},{"index":22,"sense_key":"run%2:41:00::","synset_id":"run.v.19"},{"index":23,"sense_key":"run%2:38:10::","synset_id":"run.v.26"},{"index":24,"sense_key":"run%2:35:04::","synset_id":"run.v.13"},{"index":25,"sense_key":"run%2:30:00::","synset_id":"melt.v.01"},{"index":26,"sense_key":"run%2:38:01::","synset_id":"run.v.06"},{"index":27,"sense_key":"run%2:38:02::","synset_id":"run.v.32"},{"index":28,"sense_key":"run%2:42:08::","synset_id":"run.v.23"},{"index":29,"sense_key":"run%2:36:01::","synset_id":"play.v.18"},{"index":30,"sense_key":"run%2:30:01::","synset_id":"ladder.v.01"},{"index":31,"sense_key":"run%2:35:01::","synset_id":"guide.v.05"},{"index":32,"sense_key":"run%2:33:06::","synset_id":"campaign.v.01"},{"index":33,"sense_key":"run%2:41:01::","synset_id":"run.v.24"},{"index":34,"sense_key":"run%2:42:02::","synset_id":"run.v.16"},{"index":35,"sense_key":"run%2:42:07::","synset_id":"range.v.01"},{"index":36,"sense_key":"run%2:38:06::","synset_id":"run.v.33"},{"index":37,"sense_key":"run%2:41:02::","synset_id":"operate.v.01"},{"index":38,"sense_key":"run%2:40:06::","synset_id":"run.v.25"},{"index":39,"sense_key":"run%2:38:11::","synset_id":"run.v.11"},{"index":40,"sense_key":"run%2:42:01::","synset_id":"tend.v.01"},{"index":41,"sense_key":"run%2:35:00::","synset_id":"function.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"run%2:38:00::","call_ids":["senseval3.d002.s086.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"run%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2002,"cached_input_tokens":0,"output_tokens":1045,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.04136000000000001,"input_uncached_usd":0.010010000000000002,"input_cached_usd":0.0,"output_usd":0.03135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":21.464588582981378}
{"item_id":"senseval3.d002.s086.t003","gold_sense_keys":["door%1:06:00::","door%1:06:01::"],"candidates":[{"index":1,"sense_key":"door%1:07:00::","synset_id":"door.n.03"},{"index":2,"sense_key":"door%1:06:01::","synset_id":"doorway.n.01"},{"index":3,"sense_key":"door%1:06:00::","synset_id":"door.n.01"},{"index":4,"sense_key":"door%1:06:04::","synset_id":"door.n.04"},{"index":5,"sense_key":"door%1:06:03::","synset_id":"door.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"door%1:06:01::","call_ids":["senseval3.d002.s086.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"door%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":136},"cost":{"currency":"USD","total_usd":0.007300000000000001,"input_uncached_usd":0.0027400000000000002,"input_cached_usd":0.0,"output_usd":0.00456,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9225358329713345}
{"item_id":"senseval3.d002.s086.t004","gold_sense_keys":["spend%2:42:00::"],"candidates":[{"index":1,"sense_key":"spend%2:42:00::","synset_id":"spend.v.01"},{"index":2,"sense_key":"spend%2:40:00::","synset_id":"spend.v.02"},{"index":3,"sense_key":"spend%2:40:01::","synset_id":"spend.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"spend%2:42:00::","call_ids":["senseval3.d002.s086.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"spend%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":406,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.00365,"input_uncached_usd":0.00203,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6266401670873165}
{"item_id":"senseval3.d002.s086.t005","gold_sense_keys":["next%5:00:00:succeeding:00"],"candidates":[{"index":1,"sense_key":"next%5:00:00:incoming:00","synset_id":"future.s.02"},{"index":2,"sense_key":"next%5:00:00:succeeding:00","synset_id":"following.s.02"},{"index":3,"sense_key":"next%5:00:00:close:01","synset_id":"adjacent.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"next%5:00:00:succeeding:00","call_ids":["senseval3.d002.s086.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"next%5:00:00:succeeding:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":438,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.0037800000000000004,"input_uncached_usd":0.00219,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6196934579638764}
{"item_id":"senseval3.d002.s086.t007","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":2,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":3,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":4,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"},{"index":5,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":6,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"minute%1:28:00::","call_ids":["senseval3.d002.s086.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.005045,"input_uncached_usd":0.002735,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8802604579832405}
{"item_id":"senseval3.d002.s086.t008","gold_sense_keys":["watch%2:39:00::","watch%2:39:02::","watch%2:41:00::"],"candidates":[{"index":1,"sense_key":"watch%2:39:05::","synset_id":"watch.v.03"},{"index":2,"sense_key":"watch%2:39:00::","synset_id":"watch.v.01"},{"index":3,"sense_key":"watch%2:39:02::","synset_id":"watch.v.04"},{"index":4,"sense_key":"watch%2:39:03::","synset_id":"watch.v.06"},{"index":5,"sense_key":"watch%2:32:09::","synset_id":"determine.v.08"},{"index":6,"sense_key":"watch%2:41:00::","synset_id":"watch.v.02"},{"index":7,"sense_key":"watch%2:39:01::","synset_id":"watch.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"watch%2:39:02::","call_ids":["senseval3.d002.s086.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"watch%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018875000000000003,"input_uncached_usd":0.0030650000000000004,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.798965374939144}
{"item_id":"senseval3.d002.s086.t009","gold_sense_keys":["brick%1:06:00::"],"candidates":[{"index":1,"sense_key":"brick%1:06:00::","synset_id":"brick.n.01"},{"index":2,"sense_key":"brick%1:18:00::","synset_id":"brick.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brick%1:06:00::","call_ids":["senseval3.d002.s086.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brick%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":364,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.00332,"input_uncached_usd":0.0018200000000000002,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4540068749338388}
{"item_id":"senseval3.d002.s086.t011","gold_sense_keys":["foot%1:08:01::"],"candidates":[{"index":1,"sense_key":"foot%1:05:01::","synset_id":"animal_foot.n.01"},{"index":2,"sense_key":"foot%1:10:00::","synset_id":"metrical_foot.n.01"},{"index":3,"sense_key":"foot%1:15:00::","synset_id":"foot.n.03"},{"index":4,"sense_key":"foot%1:23:00::","synset_id":"foot.n.02"},{"index":5,"sense_key":"foot%1:06:00::","synset_id":"foundation.n.03"},{"index":6,"sense_key":"foot%1:08:01::","synset_id":"foot.n.01"},{"index":7,"sense_key":"foot%1:04:00::","synset_id":"foot.n.07"},{"index":8,"sense_key":"foot%1:06:01::","synset_id":"foot.n.11"},{"index":9,"sense_key":"foot%1:05:00::","synset_id":"foot.n.06"},{"index":10,"sense_key":"foot%1:18:00::","synset_id":"foot.n.08"},{"index":11,"sense_key":"foot%1:14:00::","synset_id":"infantry.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"foot%1:08:01::","call_ids":["senseval3.d002.s086.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"foot%1:08:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.005485,"input_uncached_usd":0.0038050000000000002,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.020775958080776}
{"item_id":"senseval3.d002.s086.t013","gold_sense_keys":["flower%1:20:00::"],"candidates":[{"index":1,"sense_key":"flower%1:20:00::","synset_id":"flower.n.01"},{"index":2,"sense_key":"flower%1:28:00::","synset_id":"flower.n.03"},{"index":3,"sense_key":"flower%1:20:02::","synset_id":"flower.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"flower%1:20:00::","call_ids":["senseval3.d002.s086.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"flower%1:20:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":354,"reasoning_output_tokens":339},"cost":{"currency":"USD","total_usd":0.012655000000000001,"input_uncached_usd":0.0020350000000000004,"input_cached_usd":0.0,"output_usd":0.010620000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.425089124939404}
{"item_id":"senseval3.d002.s086.t014","gold_sense_keys":["wave%2:38:00::"],"candidates":[{"index":1,"sense_key":"wave%2:35:00::","synset_id":"brandish.v.01"},{"index":2,"sense_key":"wave%2:29:00::","synset_id":"wave.v.05"},{"index":3,"sense_key":"wave%2:38:00::","synset_id":"roll.v.11"},{"index":4,"sense_key":"wave%2:35:01::","synset_id":"curl.v.04"},{"index":5,"sense_key":"wave%2:32:00::","synset_id":"beckon.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"wave%2:38:00::","call_ids":["senseval3.d002.s086.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"wave%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.004750000000000001,"input_uncached_usd":0.00253,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.076399500016123}
{"item_id":"senseval3.d002.s087.t000","gold_sense_keys":["amazing%5:00:00:surprising:00"],"candidates":[{"index":1,"sense_key":"amazing%5:00:00:impressive:00","synset_id":"amazing.s.02"},{"index":2,"sense_key":"amazing%5:00:00:surprising:00","synset_id":"amazing.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"amazing%5:00:00:surprising:00","call_ids":["senseval3.d002.s087.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"amazing%5:00:00:surprising:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":405,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.0043950000000000005,"input_uncached_usd":0.0020250000000000003,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.047242374974303}
{"item_id":"senseval3.d002.s087.t001","gold_sense_keys":["do%2:36:02::"],"candidates":[{"index":1,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":2,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":3,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":4,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":5,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":6,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":7,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":8,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":9,"sense_key":"do%2:38:00::","synset_id":"do.v.13"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":12,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":13,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"do%2:36:02::","call_ids":["senseval3.d002.s087.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"do%2:36:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":887,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":138},"cost":{"currency":"USD","total_usd":0.009055,"input_uncached_usd":0.004435000000000001,"input_cached_usd":0.0,"output_usd":0.00462,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.62218487507198}
{"item_id":"senseval3.d002.s087.t002","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval3.d002.s087.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":125},"cost":{"currency":"USD","total_usd":0.006150000000000001,"input_uncached_usd":0.0019500000000000001,"input_cached_usd":0.0,"output_usd":0.0042,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.977531792013906}
{"item_id":"senseval3.d002.s087.t004","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:09:00::","synset_id":"one.n.02"},{"index":2,"sense_key":"one%1:23:00::","synset_id":"one.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval3.d002.s087.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":160,"reasoning_output_tokens":145},"cost":{"currency":"USD","total_usd":0.00675,"input_uncached_usd":0.0019500000000000001,"input_cached_usd":0.0,"output_usd":0.0048000000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.228346333024092}
{"item_id":"senseval3.d002.s088.t000","gold_sense_keys":["look%2:29:00::","look%2:39:01::"],"candidates":[{"index":1,"sense_key":"look%2:31:02::","synset_id":"count.v.08"},{"index":2,"sense_key":"look%2:39:01::","synset_id":"look.v.02"},{"index":3,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":4,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":5,"sense_key":"look%2:42:00::","synset_id":"front.v.01"},{"index":6,"sense_key":"look%2:39:00::","synset_id":"look.v.01"},{"index":7,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":8,"sense_key":"look%2:39:02::","synset_id":"search.v.02"},{"index":9,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":10,"sense_key":"look%2:29:00::","synset_id":"look.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"look%2:29:00::","call_ids":["senseval3.d002.s088.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"look%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019395,"input_uncached_usd":0.0035550000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.955300582922064}
{"item_id":"senseval3.d002.s088.t001","gold_sense_keys":["calm%5:00:00:composed:00"],"candidates":[{"index":1,"sense_key":"calm%5:00:00:composed:00","synset_id":"calm.s.01"},{"index":2,"sense_key":"calm%3:00:00::","synset_id":"calm.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"calm%5:00:00:composed:00","call_ids":["senseval3.d002.s088.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"calm%5:00:00:composed:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":356,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0036700000000000005,"input_uncached_usd":0.0017800000000000001,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6947265840135515}
{"item_id":"senseval3.d002.s088.t002","gold_sense_keys":["surreal%5:00:00:unreal:02"],"candidates":[{"index":1,"sense_key":"surreal%5:00:00:unreal:02","synset_id":"dreamlike.s.01"},{"index":2,"sense_key":"surreal%5:00:00:unrealistic:00","synset_id":"phantasmagoric.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"surreal%5:00:00:unreal:02","call_ids":["senseval3.d002.s088.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"surreal%5:00:00:unreal:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.00419,"input_uncached_usd":0.002,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6009317500283942}
{"item_id":"senseval3.d002.s088.t003","gold_sense_keys":["low%3:00:02::"],"candidates":[{"index":1,"sense_key":"low%5:00:00:dejected:00","synset_id":"gloomy.s.02"},{"index":2,"sense_key":"low%3:00:02::","synset_id":"low.a.01"},{"index":3,"sense_key":"low%5:00:00:humble:00","synset_id":"broken.s.01"},{"index":4,"sense_key":"low%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":5,"sense_key":"low%5:00:00:unrefined:01","synset_id":"low.s.02"},{"index":6,"sense_key":"low%5:00:00:soft:04","synset_id":"low.s.01"},{"index":7,"sense_key":"low%5:00:00:contemptible:00","synset_id":"abject.s.01"},{"index":8,"sense_key":"low%3:00:03::","synset_id":"low.a.05"},{"index":9,"sense_key":"low%3:00:01::","synset_id":"low.a.02"},{"index":10,"sense_key":"low%5:00:00:insufficient:00","synset_id":"depleted.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"low%3:00:02::","call_ids":["senseval3.d002.s088.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"low%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":714,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.00606,"input_uncached_usd":0.0035700000000000003,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.509494125028141}
{"item_id":"senseval3.d002.s088.t004","gold_sense_keys":["level%1:07:00::"],"candidates":[{"index":1,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"},{"index":2,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":3,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":4,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":5,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":8,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"level%1:07:00::","call_ids":["senseval3.d002.s088.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"level%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":120},"cost":{"currency":"USD","total_usd":0.0072250000000000005,"input_uncached_usd":0.0031450000000000002,"input_cached_usd":0.0,"output_usd":0.00408,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8579707499593496}
{"item_id":"senseval3.d002.s088.t005","gold_sense_keys":["confusion%1:09:00::"],"candidates":[{"index":1,"sense_key":"confusion%1:09:00::","synset_id":"confusion.n.02"},{"index":2,"sense_key":"confusion%1:12:00::","synset_id":"confusion.n.03"},{"index":3,"sense_key":"confusion%1:04:01::","synset_id":"confusion.n.04"},{"index":4,"sense_key":"confusion%1:04:00::","synset_id":"confusion.n.05"},{"index":5,"sense_key":"confusion%1:26:01::","synset_id":"confusion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"confusion%1:09:00::","call_ids":["senseval3.d002.s088.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"confusion%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":186,"reasoning_output_tokens":170},"cost":{"currency":"USD","total_usd":0.00799,"input_uncached_usd":0.0024100000000000002,"input_cached_usd":0.0,"output_usd":0.00558,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6138539171079174}
{"item_id":"senseval3.d002.s088.t007","gold_sense_keys":["continue%2:30:10::"],"candidates":[{"index":1,"sense_key":"continue%2:42:01::","synset_id":"continue.v.01"},{"index":2,"sense_key":"continue%2:42:10::","synset_id":"stay.v.04"},{"index":3,"sense_key":"continue%2:38:00::","synset_id":"proceed.v.02"},{"index":4,"sense_key":"continue%2:42:12::","synset_id":"continue.v.06"},{"index":5,"sense_key":"continue%2:42:11::","synset_id":"continue.v.07"},{"index":6,"sense_key":"continue%2:42:00::","synset_id":"continue.v.03"},{"index":7,"sense_key":"continue%2:32:00::","synset_id":"continue.v.02"},{"index":8,"sense_key":"continue%2:30:10::","synset_id":"continue.v.10"},{"index":9,"sense_key":"continue%2:42:06::","synset_id":"cover.v.03"},{"index":10,"sense_key":"continue%2:41:00::","synset_id":"retain.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"continue%2:30:10::","call_ids":["senseval3.d002.s088.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"continue%2:30:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":757,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019625,"input_uncached_usd":0.003785,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.723911500070244}
{"item_id":"senseval3.d002.s091.t001","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":2,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval3.d002.s091.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"seem%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01829,"input_uncached_usd":0.0024500000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.555985542014241}
{"item_id":"senseval3.d002.s091.t003","gold_sense_keys":["night%1:28:00::"],"candidates":[{"index":1,"sense_key":"night%1:28:05::","synset_id":"night.n.06"},{"index":2,"sense_key":"night%1:26:00::","synset_id":"night.n.05"},{"index":3,"sense_key":"night%1:28:03::","synset_id":"night.n.07"},{"index":4,"sense_key":"night%1:28:04::","synset_id":"night.n.03"},{"index":5,"sense_key":"night%1:28:02::","synset_id":"night.n.04"},{"index":6,"sense_key":"night%1:18:00::","synset_id":"nox.n.01"},{"index":7,"sense_key":"night%1:28:00::","synset_id":"night.n.01"},{"index":8,"sense_key":"night%1:28:01::","synset_id":"night.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"night%1:28:00::","call_ids":["senseval3.d002.s091.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"night%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018625,"input_uncached_usd":0.002785,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.384496499900706}
{"item_id":"senseval3.d002.s091.t004","gold_sense_keys":["do%2:36:01::"],"candidates":[{"index":1,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":2,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":3,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":4,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":5,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":6,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":7,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":8,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":9,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":10,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"do%2:36:01::","call_ids":["senseval3.d002.s091.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":903,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020355,"input_uncached_usd":0.004515000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.250684292055666}
{"item_id":"senseval3.d002.s091.t007","gold_sense_keys":["sound%1:07:00::"],"candidates":[{"index":1,"sense_key":"sound%1:10:01::","synset_id":"phone.n.02"},{"index":2,"sense_key":"sound%1:11:00::","synset_id":"sound.n.04"},{"index":3,"sense_key":"sound%1:10:00::","synset_id":"audio.n.01"},{"index":4,"sense_key":"sound%1:17:00::","synset_id":"sound.n.08"},{"index":5,"sense_key":"sound%1:07:00::","synset_id":"sound.n.01"},{"index":6,"sense_key":"sound%1:17:01::","synset_id":"strait.n.01"},{"index":7,"sense_key":"sound%1:19:00::","synset_id":"sound.n.03"},{"index":8,"sense_key":"sound%1:09:00::","synset_id":"sound.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"sound%1:07:00::","call_ids":["senseval3.d002.s091.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"sound%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":454,"reasoning_output_tokens":439},"cost":{"currency":"USD","total_usd":0.016720000000000002,"input_uncached_usd":0.0031000000000000003,"input_cached_usd":0.0,"output_usd":0.01362,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.452952792053111}
{"item_id":"senseval3.d002.s091.t008","gold_sense_keys":["light%1:06:00::","light%1:19:00::"],"candidates":[{"index":1,"sense_key":"light%1:06:00::","synset_id":"light.n.02"},{"index":2,"sense_key":"light%1:26:01::","synset_id":"light.n.06"},{"index":3,"sense_key":"light%1:16:00::","synset_id":"inner_light.n.01"},{"index":4,"sense_key":"light%1:09:00::","synset_id":"light.n.10"},{"index":5,"sense_key":"light%1:09:02::","synset_id":"light.n.12"},{"index":6,"sense_key":"light%1:07:01::","synset_id":"luminosity.n.01"},{"index":7,"sense_key":"light%1:15:00::","synset_id":"light.n.05"},{"index":8,"sense_key":"light%1:19:00::","synset_id":"light.n.01"},{"index":9,"sense_key":"light%1:18:00::","synset_id":"light.n.08"},{"index":10,"sense_key":"light%1:10:00::","synset_id":"light.n.14"},{"index":11,"sense_key":"light%1:06:01::","synset_id":"lighter.n.02"},{"index":12,"sense_key":"light%1:07:02::","synset_id":"sparkle.n.01"},{"index":13,"sense_key":"light%1:26:00::","synset_id":"light.n.09"},{"index":14,"sense_key":"light%1:07:00::","synset_id":"light.n.07"},{"index":15,"sense_key":"light%1:09:01::","synset_id":"light.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"light%1:06:00::","call_ids":["senseval3.d002.s091.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"light%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":933,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020505000000000002,"input_uncached_usd":0.004665000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.497544792015105}
{"item_id":"senseval3.d002.s092.t000","gold_sense_keys":["friend%1:18:00::","friend%1:18:02::"],"candidates":[{"index":1,"sense_key":"friend%1:18:02::","synset_id":"acquaintance.n.03"},{"index":2,"sense_key":"friend%1:18:01::","synset_id":"friend.n.05"},{"index":3,"sense_key":"friend%1:18:04::","synset_id":"supporter.n.01"},{"index":4,"sense_key":"friend%1:18:00::","synset_id":"friend.n.01"},{"index":5,"sense_key":"friend%1:18:03::","synset_id":"ally.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"friend%1:18:00::","call_ids":["senseval3.d002.s092.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"friend%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":134},"cost":{"currency":"USD","total_usd":0.007040000000000001,"input_uncached_usd":0.00254,"input_cached_usd":0.0,"output_usd":0.0045000000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2914027080405504}
{"item_id":"senseval3.d002.s092.t001","gold_sense_keys":["underground%5:00:00:subsurface:00"],"candidates":[{"index":1,"sense_key":"underground%5:00:00:covert:00","synset_id":"clandestine.s.01"},{"index":2,"sense_key":"underground%5:00:00:subsurface:00","synset_id":"belowground.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"underground%5:00:00:subsurface:00","call_ids":["senseval3.d002.s092.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"underground%5:00:00:subsurface:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.004085,"input_uncached_usd":0.0020150000000000003,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8704692089231685}
{"item_id":"senseval3.d002.s092.t002","gold_sense_keys":["computer%1:06:00::"],"candidates":[{"index":1,"sense_key":"computer%1:06:00::","synset_id":"computer.n.01"},{"index":2,"sense_key":"computer%1:18:00::","synset_id":"calculator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"computer%1:06:00::","call_ids":["senseval3.d002.s092.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"computer%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":362,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.00391,"input_uncached_usd":0.0018100000000000002,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.903942832956091}
{"item_id":"senseval3.d002.s092.t003","gold_sense_keys":["center%1:06:00::"],"candidates":[{"index":1,"sense_key":"center%1:15:02::","synset_id":"center.n.11"},{"index":2,"sense_key":"center%1:14:00::","synset_id":"center.n.12"},{"index":3,"sense_key":"center%1:18:00::","synset_id":"center.n.10"},{"index":4,"sense_key":"center%1:04:01::","synset_id":"center.n.18"},{"index":5,"sense_key":"center%1:06:00::","synset_id":"center.n.03"},{"index":6,"sense_key":"center%1:18:01::","synset_id":"center.n.09"},{"index":7,"sense_key":"center%1:14:01::","synset_id":"center.n.08"},{"index":8,"sense_key":"center%1:15:00::","synset_id":"center.n.04"},{"index":9,"sense_key":"center%1:13:00::","synset_id":"center.n.14"},{"index":10,"sense_key":"center%1:18:02::","synset_id":"center.n.13"},{"index":11,"sense_key":"center%1:09:00::","synset_id":"kernel.n.03"},{"index":12,"sense_key":"center%1:06:01::","synset_id":"plaza.n.02"},{"index":13,"sense_key":"center%1:06:02::","synset_id":"center_field.n.01"},{"index":14,"sense_key":"center%1:09:01::","synset_id":"center.n.06"},{"index":15,"sense_key":"center%1:08:00::","synset_id":"center.n.07"},{"index":16,"sense_key":"center%1:15:01::","synset_id":"center.n.01"},{"index":17,"sense_key":"center%1:04:02::","synset_id":"center.n.17"},{"index":18,"sense_key":"center%1:04:03::","synset_id":"center.n.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"center%1:06:00::","call_ids":["senseval3.d002.s092.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"center%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1066,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02117,"input_uncached_usd":0.0053300000000000005,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.13078783301171}
{"item_id":"senseval3.d002.s092.t006","gold_sense_keys":["hit%2:35:05::"],"candidates":[{"index":1,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":2,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":3,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":4,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":5,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":6,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"},{"index":7,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"},{"index":8,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":9,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":10,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"},{"index":11,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":12,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"},{"index":13,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"},{"index":14,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":15,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"},{"index":16,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":17,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"hit%2:35:05::","call_ids":["senseval3.d002.s092.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"hit%2:35:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1018,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":147},"cost":{"currency":"USD","total_usd":0.00995,"input_uncached_usd":0.005090000000000001,"input_cached_usd":0.0,"output_usd":0.00486,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2824611249379814}
{"item_id":"senseval3.d002.s093.t001","gold_sense_keys":["computer%1:06:00::"],"candidates":[{"index":1,"sense_key":"computer%1:18:00::","synset_id":"calculator.n.01"},{"index":2,"sense_key":"computer%1:06:00::","synset_id":"computer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"computer%1:06:00::","call_ids":["senseval3.d002.s093.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"computer%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":368,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.00379,"input_uncached_usd":0.00184,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6247419159626588}
{"item_id":"senseval3.d002.s093.t005","gold_sense_keys":["slide%2:38:01::"],"candidates":[{"index":1,"sense_key":"slide%2:38:00::","synset_id":"slither.v.01"},{"index":2,"sense_key":"slide%2:38:01::","synset_id":"skid.v.04"},{"index":3,"sense_key":"slide%2:38:02::","synset_id":"slide.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"slide%2:38:02::","call_ids":["senseval3.d002.s093.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"slide%2:38:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01793,"input_uncached_usd":0.0020900000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.218323624925688}
{"item_id":"senseval3.d002.s093.t006","gold_sense_keys":["floor%1:06:00::"],"candidates":[{"index":1,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":2,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"},{"index":3,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"},{"index":4,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"},{"index":5,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"},{"index":6,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":7,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":8,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"},{"index":9,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":10,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"floor%1:06:00::","call_ids":["senseval3.d002.s093.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"floor%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":683,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.005305,"input_uncached_usd":0.003415,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7062797080725431}
{"item_id":"senseval3.d002.s094.t000","gold_sense_keys":["today%1:28:01::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"today%1:28:01::","call_ids":["senseval3.d002.s094.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"today%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":350,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.00391,"input_uncached_usd":0.00175,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7123689580475911}
{"item_id":"senseval3.d002.s094.t002","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"},{"index":2,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":3,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":4,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval3.d002.s094.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.00456,"input_uncached_usd":0.0022500000000000003,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4397760410793126}
{"item_id":"senseval3.d002.s094.t003","gold_sense_keys":["realize%2:31:00::","realize%2:31:01::"],"candidates":[{"index":1,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"},{"index":2,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"},{"index":3,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"},{"index":4,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"},{"index":5,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":6,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"realize%2:31:01::","call_ids":["senseval3.d002.s094.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"realize%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":192,"reasoning_output_tokens":176},"cost":{"currency":"USD","total_usd":0.0086,"input_uncached_usd":0.00284,"input_cached_usd":0.0,"output_usd":0.00576,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.272813708987087}
{"item_id":"senseval3.d002.s094.t004","gold_sense_keys":["hard%3:00:06::"],"candidates":[{"index":1,"sense_key":"hard%5:00:00:alcoholic:00","synset_id":"hard.s.04"},{"index":2,"sense_key":"hard%5:00:01:effortful:00","synset_id":"arduous.s.01"},{"index":3,"sense_key":"hard%3:00:01::","synset_id":"hard.a.03"},{"index":4,"sense_key":"hard%5:00:00:bad:00","synset_id":"hard.s.05"},{"index":5,"sense_key":"hard%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":6,"sense_key":"hard%3:00:05::","synset_id":"hard.a.07"},{"index":7,"sense_key":"hard%5:00:00:indulgent:00","synset_id":"intemperate.s.01"},{"index":8,"sense_key":"hard%5:00:00:stale:00","synset_id":"hard.s.06"},{"index":9,"sense_key":"hard%3:00:00::","synset_id":"unvoiced.a.01"},{"index":10,"sense_key":"hard%3:00:06::","synset_id":"difficult.a.01"},{"index":11,"sense_key":"hard%3:00:02::","synset_id":"hard.a.02"},{"index":12,"sense_key":"hard%3:00:03::","synset_id":"hard.a.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"hard%3:00:06::","call_ids":["senseval3.d002.s094.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"hard%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":800,"cached_input_tokens":0,"output_tokens":171,"reasoning_output_tokens":155},"cost":{"currency":"USD","total_usd":0.009130000000000001,"input_uncached_usd":0.004,"input_cached_usd":0.0,"output_usd":0.00513,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6877471250481904}
{"item_id":"senseval3.d002.s094.t005","gold_sense_keys":["life%1:09:00::"],"candidates":[{"index":1,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":2,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":3,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":4,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":5,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":6,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"},{"index":7,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":8,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":9,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":10,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":11,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":12,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":13,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":14,"sense_key":"life%1:09:00::","synset_id":"life.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"life%1:09:00::","call_ids":["senseval3.d002.s094.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"life%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":840,"cached_input_tokens":0,"output_tokens":237,"reasoning_output_tokens":221},"cost":{"currency":"USD","total_usd":0.01131,"input_uncached_usd":0.004200000000000001,"input_cached_usd":0.0,"output_usd":0.00711,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.492288167006336}
{"item_id":"senseval3.d002.s097.t000","gold_sense_keys":["get%2:38:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":2,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":3,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":4,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":5,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":6,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":7,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":8,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":9,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":10,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":11,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":12,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":13,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":14,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":15,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":16,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"},{"index":17,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":18,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":19,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":20,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":21,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":22,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":23,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":24,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":25,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":26,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":27,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":28,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":29,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":30,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":31,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":32,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":33,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":34,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":35,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":36,"sense_key":"get%2:31:03::","synset_id":"get.v.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":30,"chosen_sense_key":"get%2:38:00::","call_ids":["senseval3.d002.s097.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":30,"predicted_sense_key":"get%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1893,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.011595000000000001,"input_uncached_usd":0.009465000000000001,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9112308750627562}
{"item_id":"senseval3.d002.s097.t001","gold_sense_keys":["home%1:15:04::"],"candidates":[{"index":1,"sense_key":"home%1:15:03::","synset_id":"home.n.06"},{"index":2,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":3,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"},{"index":4,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":5,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"},{"index":6,"sense_key":"home%1:06:01::","synset_id":"home.n.09"},{"index":7,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":8,"sense_key":"home%1:14:00::","synset_id":"family.n.01"},{"index":9,"sense_key":"home%1:15:02::","synset_id":"home.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"home%1:15:04::","call_ids":["senseval3.d002.s097.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"home%1:15:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":197,"reasoning_output_tokens":181},"cost":{"currency":"USD","total_usd":0.00927,"input_uncached_usd":0.00336,"input_cached_usd":0.0,"output_usd":0.00591,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7903482919791713}
{"item_id":"senseval3.d002.s097.t002","gold_sense_keys":["let%2:30:01::","let%2:32:02::","let%2:41:00::"],"candidates":[{"index":1,"sense_key":"let%2:32:00::","synset_id":"let.v.02"},{"index":2,"sense_key":"let%2:41:00::","synset_id":"let.v.01"},{"index":3,"sense_key":"let%2:30:01::","synset_id":"get.v.03"},{"index":4,"sense_key":"let%2:40:00::","synset_id":"lease.v.03"},{"index":5,"sense_key":"let%2:32:02::","synset_id":"permit.v.01"},{"index":6,"sense_key":"let%2:30:00::","synset_id":"let.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"let%2:41:00::","call_ids":["senseval3.d002.s097.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"let%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":3118,"reasoning_output_tokens":3071},"cost":{"currency":"USD","total_usd":0.096205,"input_uncached_usd":0.0026650000000000003,"input_cached_usd":0.0,"output_usd":0.09354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":60.41236812493298}
{"item_id":"senseval3.d002.s097.t003","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":2,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":3,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":6,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"},{"index":7,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dog%1:05:00::","call_ids":["senseval3.d002.s097.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.004525,"input_uncached_usd":0.0029950000000000003,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3569936669664457}
{"item_id":"senseval3.d002.s097.t004","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":2,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":3,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":4,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":5,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":6,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":7,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":8,"sense_key":"house%1:06:01::","synset_id":"house.n.12"},{"index":9,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":10,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":11,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":12,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s097.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":734,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.0055,"input_uncached_usd":0.00367,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8889032088918611}
{"item_id":"senseval3.d002.s097.t005","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d002.s097.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":519,"reasoning_output_tokens":503},"cost":{"currency":"USD","total_usd":0.017875000000000002,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.01557,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.751069583930075}
{"item_id":"senseval3.d002.s097.t006","gold_sense_keys":["sound%1:09:00::"],"candidates":[{"index":1,"sense_key":"sound%1:11:00::","synset_id":"sound.n.04"},{"index":2,"sense_key":"sound%1:09:00::","synset_id":"sound.n.02"},{"index":3,"sense_key":"sound%1:17:00::","synset_id":"sound.n.08"},{"index":4,"sense_key":"sound%1:19:00::","synset_id":"sound.n.03"},{"index":5,"sense_key":"sound%1:10:01::","synset_id":"phone.n.02"},{"index":6,"sense_key":"sound%1:07:00::","synset_id":"sound.n.01"},{"index":7,"sense_key":"sound%1:17:01::","synset_id":"strait.n.01"},{"index":8,"sense_key":"sound%1:10:00::","synset_id":"audio.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sound%1:09:00::","call_ids":["senseval3.d002.s097.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sound%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.034249999999999996,"input_uncached_usd":0.00287,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":20.238217999925837}
{"item_id":"senseval3.d002.s097.t007","gold_sense_keys":["head%1:08:00::"],"candidates":[{"index":1,"sense_key":"head%1:18:02::","synset_id":"head.n.15"},{"index":2,"sense_key":"head%1:17:02::","synset_id":"promontory.n.01"},{"index":3,"sense_key":"head%1:06:03::","synset_id":"head.n.29"},{"index":4,"sense_key":"head%1:06:01::","synset_id":"drumhead.n.01"},{"index":5,"sense_key":"head%1:10:02::","synset_id":"question.n.02"},{"index":6,"sense_key":"head%1:18:00::","synset_id":"head.n.04"},{"index":7,"sense_key":"head%1:06:00::","synset_id":"head.n.31"},{"index":8,"sense_key":"head%1:11:00::","synset_id":"headway.n.02"},{"index":9,"sense_key":"head%1:15:02::","synset_id":"forefront.n.01"},{"index":10,"sense_key":"head%1:11:01::","synset_id":"pass.n.09"},{"index":11,"sense_key":"head%1:08:02::","synset_id":"head.n.26"},{"index":12,"sense_key":"head%1:09:00::","synset_id":"mind.n.01"},{"index":13,"sense_key":"head%1:08:00::","synset_id":"head.n.01"},{"index":14,"sense_key":"head%1:15:01::","synset_id":"fountainhead.n.02"},{"index":15,"sense_key":"head%1:06:02::","synset_id":"head.n.30"},{"index":16,"sense_key":"head%1:10:03::","synset_id":"head.n.09"},{"index":17,"sense_key":"head%1:08:01::","synset_id":"head.n.25"},{"index":18,"sense_key":"head%1:18:03::","synset_id":"head.n.14"},{"index":19,"sense_key":"head%1:06:05::","synset_id":"read/write_head.n.01"},{"index":20,"sense_key":"head%1:10:01::","synset_id":"point.n.20"},{"index":21,"sense_key":"head%1:17:01::","synset_id":"head.n.17"},{"index":22,"sense_key":"head%1:26:00::","synset_id":"head.n.10"},{"index":23,"sense_key":"head%1:23:00::","synset_id":"head.n.11"},{"index":24,"sense_key":"head%1:04:00::","synset_id":"oral_sex.n.01"},{"index":25,"sense_key":"head%1:18:01::","synset_id":"principal.n.02"},{"index":26,"sense_key":"head%1:10:00::","synset_id":"heading.n.01"},{"index":27,"sense_key":"head%1:19:00::","synset_id":"head.n.06"},{"index":28,"sense_key":"head%1:17:00::","synset_id":"head.n.18"},{"index":29,"sense_key":"head%1:14:00::","synset_id":"head.n.05"},{"index":30,"sense_key":"head%1:20:00::","synset_id":"capitulum.n.01"},{"index":31,"sense_key":"head%1:06:04::","synset_id":"head.n.28"},{"index":32,"sense_key":"head%1:15:00::","synset_id":"head.n.07"},{"index":33,"sense_key":"head%1:05:00::","synset_id":"head.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"head%1:08:00::","call_ids":["senseval3.d002.s097.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"head%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1543,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.009635000000000001,"input_uncached_usd":0.0077150000000000005,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.993885666015558}
{"item_id":"senseval3.d002.s097.t009","gold_sense_keys":["walk%2:38:00::"],"candidates":[{"index":1,"sense_key":"walk%2:38:02::","synset_id":"walk.v.08"},{"index":2,"sense_key":"walk%2:33:01::","synset_id":"walk.v.05"},{"index":3,"sense_key":"walk%2:33:00::","synset_id":"walk.v.03"},{"index":4,"sense_key":"walk%2:38:00::","synset_id":"walk.v.01"},{"index":5,"sense_key":"walk%2:38:03::","synset_id":"walk.v.04"},{"index":6,"sense_key":"walk%2:38:04::","synset_id":"walk.v.10"},{"index":7,"sense_key":"walk%2:41:00::","synset_id":"walk.v.06"},{"index":8,"sense_key":"walk%2:41:01::","synset_id":"walk.v.07"},{"index":9,"sense_key":"walk%2:38:05::","synset_id":"walk.v.09"},{"index":10,"sense_key":"walk%2:38:01::","synset_id":"walk.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"walk%2:38:00::","call_ids":["senseval3.d002.s097.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"walk%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.0049250000000000006,"input_uncached_usd":0.0030050000000000003,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.899389542057179}
{"item_id":"senseval3.d002.s097.t010","gold_sense_keys":["roof%1:06:00::"],"candidates":[{"index":1,"sense_key":"roof%1:06:01::","synset_id":"roof.n.02"},{"index":2,"sense_key":"roof%1:06:00::","synset_id":"roof.n.01"},{"index":3,"sense_key":"roof%1:10:00::","synset_id":"ceiling.n.03"},{"index":4,"sense_key":"roof%1:17:00::","synset_id":"roof.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"roof%1:06:00::","call_ids":["senseval3.d002.s097.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"roof%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.005025,"input_uncached_usd":0.002085,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.354216124978848}
{"item_id":"senseval3.d002.s098.t000","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":2,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":3,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d002.s098.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018125,"input_uncached_usd":0.002285,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.158459792030044}
{"item_id":"senseval3.d002.s098.t001","gold_sense_keys":["car%1:06:00::"],"candidates":[{"index":1,"sense_key":"car%1:06:00::","synset_id":"car.n.01"},{"index":2,"sense_key":"car%1:06:04::","synset_id":"cable_car.n.01"},{"index":3,"sense_key":"car%1:06:01::","synset_id":"car.n.02"},{"index":4,"sense_key":"car%1:06:03::","synset_id":"car.n.03"},{"index":5,"sense_key":"car%1:06:02::","synset_id":"car.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"car%1:06:00::","call_ids":["senseval3.d002.s098.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"car%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00435,"input_uncached_usd":0.0024000000000000002,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1971084169344977}
{"item_id":"senseval3.d002.s098.t002","gold_sense_keys":["bounce%2:38:02::"],"candidates":[{"index":1,"sense_key":"bounce%2:38:01::","synset_id":"bounce.v.05"},{"index":2,"sense_key":"bounce%2:35:03::","synset_id":"bounce.v.02"},{"index":3,"sense_key":"bounce%2:40:01::","synset_id":"bounce.v.06"},{"index":4,"sense_key":"bounce%2:38:00::","synset_id":"bounce.v.01"},{"index":5,"sense_key":"bounce%2:35:00::","synset_id":"bounce.v.07"},{"index":6,"sense_key":"bounce%2:38:02::","synset_id":"bounce.v.03"},{"index":7,"sense_key":"bounce%2:40:00::","synset_id":"bounce.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"bounce%2:38:02::","call_ids":["senseval3.d002.s098.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"bounce%2:38:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004535000000000001,"input_uncached_usd":0.0024950000000000003,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7183082500705495}
{"item_id":"senseval3.d002.s099.t000","gold_sense_keys":["realize%2:31:00::"],"candidates":[{"index":1,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"},{"index":2,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"},{"index":3,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":4,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"},{"index":5,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"},{"index":6,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"realize%2:31:00::","call_ids":["senseval3.d002.s099.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"realize%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":258,"reasoning_output_tokens":242},"cost":{"currency":"USD","total_usd":0.010655000000000001,"input_uncached_usd":0.002915,"input_cached_usd":0.0,"output_usd":0.00774,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.099534749984741}
{"item_id":"senseval3.d002.s099.t001","gold_sense_keys":["happen%2:30:00::"],"candidates":[{"index":1,"sense_key":"happen%2:30:02::","synset_id":"happen.v.02"},{"index":2,"sense_key":"happen%2:30:00::","synset_id":"happen.v.01"},{"index":3,"sense_key":"happen%2:40:12::","synset_id":"find.v.01"},{"index":4,"sense_key":"happen%2:41:00::","synset_id":"happen.v.03"},{"index":5,"sense_key":"happen%2:30:01::","synset_id":"happen.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"happen%2:30:00::","call_ids":["senseval3.d002.s099.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"happen%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":542,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":130},"cost":{"currency":"USD","total_usd":0.007090000000000001,"input_uncached_usd":0.00271,"input_cached_usd":0.0,"output_usd":0.00438,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5154397919541225}
{"item_id":"senseval3.d002.s099.t002","gold_sense_keys":["scream%2:32:01::"],"candidates":[{"index":1,"sense_key":"scream%2:32:08::","synset_id":"yell.v.02"},{"index":2,"sense_key":"scream%2:32:01::","synset_id":"shout.v.02"},{"index":3,"sense_key":"scream%2:39:00::","synset_id":"scream.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"scream%2:32:01::","call_ids":["senseval3.d002.s099.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"scream%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":431,"cached_input_tokens":0,"output_tokens":216,"reasoning_output_tokens":200},"cost":{"currency":"USD","total_usd":0.008635,"input_uncached_usd":0.0021550000000000002,"input_cached_usd":0.0,"output_usd":0.0064800000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.450292333960533}
{"item_id":"senseval3.d002.s099.t003","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":2,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":3,"sense_key":"house%1:06:01::","synset_id":"house.n.12"},{"index":4,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":5,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":6,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":9,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":10,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":11,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":12,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s099.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.007015,"input_uncached_usd":0.0038050000000000002,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3353785830549896}
{"item_id":"senseval3.d002.s099.t004","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":2,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":3,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"},{"index":4,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":5,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":6,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":7,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"dog%1:05:00::","call_ids":["senseval3.d002.s099.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":28},"cost":{"currency":"USD","total_usd":0.004445,"input_uncached_usd":0.003125,"input_cached_usd":0.0,"output_usd":0.00132,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2720417089294642}
{"item_id":"senseval3.d002.s100.t001","gold_sense_keys":["door%1:06:00::"],"candidates":[{"index":1,"sense_key":"door%1:06:01::","synset_id":"doorway.n.01"},{"index":2,"sense_key":"door%1:06:00::","synset_id":"door.n.01"},{"index":3,"sense_key":"door%1:07:00::","synset_id":"door.n.03"},{"index":4,"sense_key":"door%1:06:04::","synset_id":"door.n.04"},{"index":5,"sense_key":"door%1:06:03::","synset_id":"door.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"door%1:06:00::","call_ids":["senseval3.d002.s100.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"door%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":554,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.00511,"input_uncached_usd":0.0027700000000000003,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.322094167000614}
{"item_id":"senseval3.d002.s100.t007","gold_sense_keys":["foot%1:23:00::"],"candidates":[{"index":1,"sense_key":"foot%1:23:00::","synset_id":"foot.n.02"},{"index":2,"sense_key":"foot%1:05:01::","synset_id":"animal_foot.n.01"},{"index":3,"sense_key":"foot%1:18:00::","synset_id":"foot.n.08"},{"index":4,"sense_key":"foot%1:14:00::","synset_id":"infantry.n.01"},{"index":5,"sense_key":"foot%1:15:00::","synset_id":"foot.n.03"},{"index":6,"sense_key":"foot%1:06:01::","synset_id":"foot.n.11"},{"index":7,"sense_key":"foot%1:06:00::","synset_id":"foundation.n.03"},{"index":8,"sense_key":"foot%1:10:00::","synset_id":"metrical_foot.n.01"},{"index":9,"sense_key":"foot%1:05:00::","synset_id":"foot.n.06"},{"index":10,"sense_key":"foot%1:08:01::","synset_id":"foot.n.01"},{"index":11,"sense_key":"foot%1:04:00::","synset_id":"foot.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"foot%1:23:00::","call_ids":["senseval3.d002.s100.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"foot%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00575,"input_uncached_usd":0.0038300000000000005,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.07120004191529}
{"item_id":"senseval3.d002.s100.t008","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":2,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":3,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":6,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"dog%1:05:00::","call_ids":["senseval3.d002.s100.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.005220000000000001,"input_uncached_usd":0.0032400000000000003,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2499880000250414}
{"item_id":"senseval3.d002.s100.t009","gold_sense_keys":["come%2:38:00::","come%2:38:04::"],"candidates":[{"index":1,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":2,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":3,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":4,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":5,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":7,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":8,"sense_key":"come%2:31:13::","synset_id":"come.v.21"},{"index":9,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":10,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":13,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":14,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":15,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":20,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":21,"sense_key":"come%2:30:13::","synset_id":"come.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"come%2:38:00::","call_ids":["senseval3.d002.s100.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"come%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1182,"cached_input_tokens":0,"output_tokens":279,"reasoning_output_tokens":264},"cost":{"currency":"USD","total_usd":0.014280000000000001,"input_uncached_usd":0.00591,"input_cached_usd":0.0,"output_usd":0.00837,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.118353833095171}
{"item_id":"senseval3.d002.s100.t011","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"dog%1:05:00::","call_ids":["senseval3.d002.s100.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.00567,"input_uncached_usd":0.0032400000000000003,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.409835916943848}
{"item_id":"senseval3.d002.s100.t012","gold_sense_keys":["stand%2:35:00::","stand%2:42:03::"],"candidates":[{"index":1,"sense_key":"stand%2:35:00::","synset_id":"stand.v.01"},{"index":2,"sense_key":"stand%2:42:06::","synset_id":"stand.v.07"},{"index":3,"sense_key":"stand%2:35:10::","synset_id":"stand.v.12"},{"index":4,"sense_key":"stand%2:42:02::","synset_id":"stand.v.08"},{"index":5,"sense_key":"stand%2:35:01::","synset_id":"stand.v.10"},{"index":6,"sense_key":"stand%2:33:00::","synset_id":"resist.v.04"},{"index":7,"sense_key":"stand%2:42:01::","synset_id":"stand.v.09"},{"index":8,"sense_key":"stand%2:42:04::","synset_id":"stand.v.02"},{"index":9,"sense_key":"stand%2:31:01::","synset_id":"stand.v.06"},{"index":10,"sense_key":"stand%2:42:03::","synset_id":"stand.v.03"},{"index":11,"sense_key":"stand%2:31:00::","synset_id":"digest.v.03"},{"index":12,"sense_key":"stand%2:42:00::","synset_id":"stand.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stand%2:35:00::","call_ids":["senseval3.d002.s100.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stand%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":1218,"reasoning_output_tokens":1190},"cost":{"currency":"USD","total_usd":0.040375,"input_uncached_usd":0.0038350000000000003,"input_cached_usd":0.0,"output_usd":0.03654,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":18.883597249980085}
{"item_id":"senseval3.d002.s100.t014","gold_sense_keys":["watch%2:39:00::","watch%2:39:02::","watch%2:41:00::"],"candidates":[{"index":1,"sense_key":"watch%2:32:09::","synset_id":"determine.v.08"},{"index":2,"sense_key":"watch%2:39:00::","synset_id":"watch.v.01"},{"index":3,"sense_key":"watch%2:39:02::","synset_id":"watch.v.04"},{"index":4,"sense_key":"watch%2:39:01::","synset_id":"watch.v.05"},{"index":5,"sense_key":"watch%2:39:03::","synset_id":"watch.v.06"},{"index":6,"sense_key":"watch%2:39:05::","synset_id":"watch.v.03"},{"index":7,"sense_key":"watch%2:41:00::","synset_id":"watch.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"watch%2:39:02::","call_ids":["senseval3.d002.s100.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"watch%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":1774,"reasoning_output_tokens":1740},"cost":{"currency":"USD","total_usd":0.056315000000000004,"input_uncached_usd":0.003095,"input_cached_usd":0.0,"output_usd":0.05322,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":37.67169858305715}
{"item_id":"senseval3.d002.s100.t015","gold_sense_keys":["outside%3:00:04::"],"candidates":[{"index":1,"sense_key":"outside%5:00:00:extramural:00","synset_id":"outside.s.03"},{"index":2,"sense_key":"outside%5:00:00:outer:00","synset_id":"outside.s.07"},{"index":3,"sense_key":"outside%3:00:04::","synset_id":"outdoor.a.01"},{"index":4,"sense_key":"outside%5:00:00:foreign:02","synset_id":"external.s.02"},{"index":5,"sense_key":"outside%5:00:00:exterior:00","synset_id":"outside.s.04"},{"index":6,"sense_key":"outside%5:00:00:extrinsic:00","synset_id":"external.s.01"},{"index":7,"sense_key":"outside%5:00:00:inaccurate:00","synset_id":"away.s.02"},{"index":8,"sense_key":"outside%5:00:00:external:00","synset_id":"outside.s.02"},{"index":9,"sense_key":"outside%3:00:00::","synset_id":"outside.a.01"},{"index":10,"sense_key":"outside%5:00:00:unlikely:00","synset_id":"outside.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"outside%3:00:04::","call_ids":["senseval3.d002.s100.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"outside%3:00:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":135},"cost":{"currency":"USD","total_usd":0.00818,"input_uncached_usd":0.0036500000000000005,"input_cached_usd":0.0,"output_usd":0.00453,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.451683708000928}
{"item_id":"senseval3.d002.s100.t016","gold_sense_keys":["dance%2:38:02::"],"candidates":[{"index":1,"sense_key":"dance%2:38:01::","synset_id":"dance.v.01"},{"index":2,"sense_key":"dance%2:38:02::","synset_id":"dance.v.03"},{"index":3,"sense_key":"dance%2:36:00::","synset_id":"dance.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dance%2:38:02::","call_ids":["senseval3.d002.s100.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dance%2:38:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":86},"cost":{"currency":"USD","total_usd":0.0052450000000000005,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.00303,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0505670830607414}
{"item_id":"senseval3.d002.s101.t000","gold_sense_keys":["realize%2:31:00::"],"candidates":[{"index":1,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"},{"index":2,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"},{"index":3,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"},{"index":4,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":5,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"},{"index":6,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"realize%2:31:01::","call_ids":["senseval3.d002.s101.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"realize%2:31:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018885,"input_uncached_usd":0.0030450000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.839620583923534}
{"item_id":"senseval3.d002.s101.t001","gold_sense_keys":["go%2:38:00::"],"candidates":[{"index":1,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":2,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":3,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":4,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":5,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":6,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":7,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":8,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":9,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"},{"index":10,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"},{"index":11,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":12,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":13,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":14,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":15,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":16,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":17,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":18,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":19,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":20,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"},{"index":21,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":22,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":23,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"},{"index":24,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":25,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":26,"sense_key":"go%2:38:01::","synset_id":"go.v.03"},{"index":27,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":28,"sense_key":"go%2:30:08::","synset_id":"go.v.09"},{"index":29,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":30,"sense_key":"go%2:42:08::","synset_id":"go.v.25"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"go%2:38:00::","call_ids":["senseval3.d002.s101.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"go%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1711,"cached_input_tokens":0,"output_tokens":282,"reasoning_output_tokens":266},"cost":{"currency":"USD","total_usd":0.017015000000000002,"input_uncached_usd":0.008555,"input_cached_usd":0.0,"output_usd":0.00846,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.110148415900767}
{"item_id":"senseval3.d002.s101.t002","gold_sense_keys":["stand%2:35:00::","stand%2:42:03::"],"candidates":[{"index":1,"sense_key":"stand%2:33:00::","synset_id":"resist.v.04"},{"index":2,"sense_key":"stand%2:42:03::","synset_id":"stand.v.03"},{"index":3,"sense_key":"stand%2:42:02::","synset_id":"stand.v.08"},{"index":4,"sense_key":"stand%2:31:00::","synset_id":"digest.v.03"},{"index":5,"sense_key":"stand%2:42:01::","synset_id":"stand.v.09"},{"index":6,"sense_key":"stand%2:35:00::","synset_id":"stand.v.01"},{"index":7,"sense_key":"stand%2:42:04::","synset_id":"stand.v.02"},{"index":8,"sense_key":"stand%2:35:10::","synset_id":"stand.v.12"},{"index":9,"sense_key":"stand%2:31:01::","synset_id":"stand.v.06"},{"index":10,"sense_key":"stand%2:35:01::","synset_id":"stand.v.10"},{"index":11,"sense_key":"stand%2:42:00::","synset_id":"stand.v.04"},{"index":12,"sense_key":"stand%2:42:06::","synset_id":"stand.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"stand%2:35:00::","call_ids":["senseval3.d002.s101.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"stand%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":1163,"reasoning_output_tokens":1135},"cost":{"currency":"USD","total_usd":0.038745,"input_uncached_usd":0.0038550000000000004,"input_cached_usd":0.0,"output_usd":0.034890000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":22.97958608402405}
{"item_id":"senseval3.d002.s101.t004","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":2,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":3,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":4,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":5,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":6,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":7,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":8,"sense_key":"house%1:06:01::","synset_id":"house.n.12"},{"index":9,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":12,"sense_key":"house%1:06:00::","synset_id":"house.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s101.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":786,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.005730000000000001,"input_uncached_usd":0.00393,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6811504579382017}
{"item_id":"senseval3.d002.s101.t005","gold_sense_keys":["wait%2:42:00::"],"candidates":[{"index":1,"sense_key":"wait%2:41:00::","synset_id":"wait.v.04"},{"index":2,"sense_key":"wait%2:42:01::","synset_id":"wait.v.02"},{"index":3,"sense_key":"wait%2:42:00::","synset_id":"wait.v.01"},{"index":4,"sense_key":"wait%2:31:00::","synset_id":"expect.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"wait%2:42:00::","call_ids":["senseval3.d002.s101.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"wait%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":95},"cost":{"currency":"USD","total_usd":0.005775000000000001,"input_uncached_usd":0.002445,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6142431249609217}
{"item_id":"senseval3.d002.s101.t006","gold_sense_keys":["pray%2:32:00::"],"candidates":[{"index":1,"sense_key":"pray%2:32:00::","synset_id":"pray.v.01"},{"index":2,"sense_key":"pray%2:32:01::","synset_id":"beg.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pray%2:32:00::","call_ids":["senseval3.d002.s101.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pray%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":235,"reasoning_output_tokens":220},"cost":{"currency":"USD","total_usd":0.00903,"input_uncached_usd":0.00198,"input_cached_usd":0.0,"output_usd":0.00705,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.888974333065562}
{"item_id":"senseval3.d002.s101.t007","gold_sense_keys":["come%2:38:04::"],"candidates":[{"index":1,"sense_key":"come%2:31:13::","synset_id":"come.v.21"},{"index":2,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":3,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":4,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":5,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":7,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":8,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":9,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":10,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":11,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":12,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":13,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":14,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":15,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":16,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":17,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":18,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":19,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":20,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":21,"sense_key":"come%2:42:12::","synset_id":"total.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"come%2:38:04::","call_ids":["senseval3.d002.s101.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"come%2:38:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1186,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.00938,"input_uncached_usd":0.00593,"input_cached_usd":0.0,"output_usd":0.00345,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9204329160274938}
{"item_id":"senseval3.d002.s101.t008","gold_sense_keys":["home%1:15:04::"],"candidates":[{"index":1,"sense_key":"home%1:06:01::","synset_id":"home.n.09"},{"index":2,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":3,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"},{"index":4,"sense_key":"home%1:15:02::","synset_id":"home.n.03"},{"index":5,"sense_key":"home%1:14:00::","synset_id":"family.n.01"},{"index":6,"sense_key":"home%1:15:03::","synset_id":"home.n.06"},{"index":7,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":8,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":9,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"home%1:15:04::","call_ids":["senseval3.d002.s101.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"home%1:15:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":355,"reasoning_output_tokens":339},"cost":{"currency":"USD","total_usd":0.014275,"input_uncached_usd":0.003625,"input_cached_usd":0.0,"output_usd":0.01065,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.429499167017639}
{"item_id":"senseval3.d002.s101.t010","gold_sense_keys":["get%2:38:00::"],"candidates":[{"index":1,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":3,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":4,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":5,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":6,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":7,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":8,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":9,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":10,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":11,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":12,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":13,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":14,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":15,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":16,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":17,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":18,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":19,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":20,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":21,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"},{"index":22,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":23,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":24,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":25,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":26,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":27,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":28,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":29,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":30,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":31,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":32,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":33,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":34,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":35,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":36,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"get%2:38:00::","call_ids":["senseval3.d002.s101.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"get%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1946,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.011890000000000001,"input_uncached_usd":0.00973,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0368129999842495}
{"item_id":"senseval3.d002.s102.t000","gold_sense_keys":["life%1:26:02::","life%1:28:01::"],"candidates":[{"index":1,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":2,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":3,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":4,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":5,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":6,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":7,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":8,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"},{"index":9,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":10,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":11,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":12,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":13,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":14,"sense_key":"life%1:26:02::","synset_id":"life.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"life%1:28:01::","call_ids":["senseval3.d002.s102.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"life%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":896,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":146},"cost":{"currency":"USD","total_usd":0.009340000000000001,"input_uncached_usd":0.0044800000000000005,"input_cached_usd":0.0,"output_usd":0.00486,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2480411670403555}
{"item_id":"senseval3.d002.s103.t000","gold_sense_keys":["see%2:39:00::","see%2:39:03::"],"candidates":[{"index":1,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":2,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":3,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":4,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":5,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":6,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":7,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":8,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":9,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":10,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":11,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":12,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":13,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":14,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":15,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":16,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":17,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":18,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":19,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":22,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":23,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":24,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval3.d002.s103.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1449,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.023085,"input_uncached_usd":0.007245000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.327291374909692}
{"item_id":"senseval3.d002.s103.t001","gold_sense_keys":["picture%1:06:00::","picture%1:06:02::"],"candidates":[{"index":1,"sense_key":"picture%1:06:01::","synset_id":"painting.n.01"},{"index":2,"sense_key":"picture%1:06:02::","synset_id":"photograph.n.01"},{"index":3,"sense_key":"picture%1:26:00::","synset_id":"picture.n.04"},{"index":4,"sense_key":"picture%1:10:02::","synset_id":"movie.n.01"},{"index":5,"sense_key":"picture%1:06:00::","synset_id":"picture.n.01"},{"index":6,"sense_key":"picture%1:09:00::","synset_id":"mental_picture.n.01"},{"index":7,"sense_key":"picture%1:10:03::","synset_id":"word_picture.n.01"},{"index":8,"sense_key":"picture%1:10:00::","synset_id":"picture.n.05"},{"index":9,"sense_key":"picture%1:10:01::","synset_id":"video.n.01"},{"index":10,"sense_key":"picture%1:09:01::","synset_id":"picture.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"picture%1:06:02::","call_ids":["senseval3.d002.s103.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"picture%1:06:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":840,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02001,"input_uncached_usd":0.004200000000000001,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.57292016595602}
{"item_id":"senseval3.d002.s103.t002","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"},{"index":2,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":3,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":4,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":5,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":8,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":9,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":10,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"begin%2:30:00::","call_ids":["senseval3.d002.s103.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"begin%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":814,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":85},"cost":{"currency":"USD","total_usd":0.0071,"input_uncached_usd":0.004070000000000001,"input_cached_usd":0.0,"output_usd":0.00303,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4468899170169607}
{"item_id":"senseval3.d002.s103.t003","gold_sense_keys":["cry%2:29:00::"],"candidates":[{"index":1,"sense_key":"cry%2:32:07::","synset_id":"exclaim.v.01"},{"index":2,"sense_key":"cry%2:32:01::","synset_id":"shout.v.02"},{"index":3,"sense_key":"cry%2:29:00::","synset_id":"cry.v.02"},{"index":4,"sense_key":"cry%2:29:02::","synset_id":"cry.v.07"},{"index":5,"sense_key":"cry%2:34:00::","synset_id":"cry.v.05"},{"index":6,"sense_key":"cry%2:32:02::","synset_id":"cry.v.04"},{"index":7,"sense_key":"cry%2:32:03::","synset_id":"cry.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cry%2:29:00::","call_ids":["senseval3.d002.s103.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cry%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.00474,"input_uncached_usd":0.00318,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.620107916998677}
{"item_id":"senseval3.d002.s106.t000","gold_sense_keys":["evening%1:28:00::"],"candidates":[{"index":1,"sense_key":"evening%1:28:00::","synset_id":"evening.n.01"},{"index":2,"sense_key":"evening%1:28:02::","synset_id":"evening.n.03"},{"index":3,"sense_key":"evening%1:28:01::","synset_id":"evening.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"evening%1:28:02::","call_ids":["senseval3.d002.s106.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"evening%1:28:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":826},"cost":{"currency":"USD","total_usd":0.027460000000000002,"input_uncached_usd":0.00202,"input_cached_usd":0.0,"output_usd":0.02544,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.176308915950358}
{"item_id":"senseval3.d002.s106.t002","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval3.d002.s106.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00393,"input_uncached_usd":0.0022500000000000003,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6115351669723168}
{"item_id":"senseval3.d002.s106.t003","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":2,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"},{"index":5,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":6,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":7,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"dog%1:05:00::","call_ids":["senseval3.d002.s106.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.004675,"input_uncached_usd":0.0030550000000000004,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.539304459001869}
{"item_id":"senseval3.d002.s106.t004","gold_sense_keys":["walk_around%2:38:00::"],"candidates":[{"index":1,"sense_key":"walk_around%2:41:01::","synset_id":"walk_around.v.04"},{"index":2,"sense_key":"walk_around%2:38:01::","synset_id":"circumambulate.v.01"},{"index":3,"sense_key":"walk_around%2:38:00::","synset_id":"perambulate.v.02"},{"index":4,"sense_key":"walk_around%2:41:00::","synset_id":"walk_around.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"walk_around%2:38:00::","call_ids":["senseval3.d002.s106.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"walk_around%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":670,"reasoning_output_tokens":648},"cost":{"currency":"USD","total_usd":0.02233,"input_uncached_usd":0.00223,"input_cached_usd":0.0,"output_usd":0.0201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.793541416991502}
{"item_id":"senseval3.d002.s106.t005","gold_sense_keys":["drink%2:34:00::"],"candidates":[{"index":1,"sense_key":"drink%2:31:00::","synset_id":"drink_in.v.01"},{"index":2,"sense_key":"drink%2:34:00::","synset_id":"drink.v.01"},{"index":3,"sense_key":"drink%2:34:02::","synset_id":"drink.v.05"},{"index":4,"sense_key":"drink%2:34:01::","synset_id":"drink.v.02"},{"index":5,"sense_key":"drink%2:34:12::","synset_id":"toast.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"drink%2:34:01::","call_ids":["senseval3.d002.s106.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"drink%2:34:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":123},"cost":{"currency":"USD","total_usd":0.006625000000000001,"input_uncached_usd":0.002455,"input_cached_usd":0.0,"output_usd":0.00417,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8945597920101136}
{"item_id":"senseval3.d002.s109.t000","gold_sense_keys":["sit_down%2:38:00::"],"candidates":[{"index":1,"sense_key":"sit_down%2:35:03::","synset_id":"sit.v.01"},{"index":2,"sense_key":"sit_down%2:38:00::","synset_id":"sit_down.v.01"},{"index":3,"sense_key":"sit_down%2:35:00::","synset_id":"seat.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sit_down%2:38:00::","call_ids":["senseval3.d002.s109.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sit_down%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":343,"cached_input_tokens":0,"output_tokens":191,"reasoning_output_tokens":175},"cost":{"currency":"USD","total_usd":0.007445000000000001,"input_uncached_usd":0.0017150000000000002,"input_cached_usd":0.0,"output_usd":0.00573,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8684217910049483}
{"item_id":"senseval3.d002.s109.t001","gold_sense_keys":["meet%2:38:06::","meet%2:41:01::"],"candidates":[{"index":1,"sense_key":"meet%2:39:00::","synset_id":"suffer.v.10"},{"index":2,"sense_key":"meet%2:33:00::","synset_id":"meet.v.10"},{"index":3,"sense_key":"meet%2:42:01::","synset_id":"meet.v.05"},{"index":4,"sense_key":"meet%2:35:00::","synset_id":"touch.v.05"},{"index":5,"sense_key":"meet%2:34:00::","synset_id":"meet.v.04"},{"index":6,"sense_key":"meet%2:41:03::","synset_id":"meet.v.08"},{"index":7,"sense_key":"meet%2:38:01::","synset_id":"meet.v.01"},{"index":8,"sense_key":"meet%2:41:02::","synset_id":"meet.v.06"},{"index":9,"sense_key":"meet%2:42:00::","synset_id":"meet.v.11"},{"index":10,"sense_key":"meet%2:41:00::","synset_id":"meet.v.07"},{"index":11,"sense_key":"meet%2:41:01::","synset_id":"meet.v.02"},{"index":12,"sense_key":"meet%2:42:02::","synset_id":"converge.v.01"},{"index":13,"sense_key":"meet%2:38:06::","synset_id":"meet.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"meet%2:41:01::","call_ids":["senseval3.d002.s109.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"meet%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":787,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":132},"cost":{"currency":"USD","total_usd":0.008375,"input_uncached_usd":0.003935,"input_cached_usd":0.0,"output_usd":0.00444,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0596684579504654}
{"item_id":"senseval3.d002.s109.t002","gold_sense_keys":["new%3:00:00::","new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":3,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":4,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":5,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":6,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":7,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":10,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"},{"index":11,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval3.d002.s109.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":731,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019495,"input_uncached_usd":0.0036550000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.885003583971411}
{"item_id":"senseval3.d002.s109.t004","gold_sense_keys":["client%1:18:01::"],"candidates":[{"index":1,"sense_key":"client%1:18:01::","synset_id":"customer.n.01"},{"index":2,"sense_key":"client%1:06:00::","synset_id":"node.n.08"},{"index":3,"sense_key":"client%1:18:00::","synset_id":"client.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"client%1:18:01::","call_ids":["senseval3.d002.s109.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"client%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":334,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0035600000000000002,"input_uncached_usd":0.00167,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6579388750251383}
{"item_id":"senseval3.d002.s109.t005","gold_sense_keys":["couple%1:14:01::"],"candidates":[{"index":1,"sense_key":"couple%1:14:00::","synset_id":"couple.n.01"},{"index":2,"sense_key":"couple%1:14:01::","synset_id":"couple.n.02"},{"index":3,"sense_key":"couple%1:23:01::","synset_id":"couple.n.03"},{"index":4,"sense_key":"couple%1:23:00::","synset_id":"couple.n.04"},{"index":5,"sense_key":"couple%1:17:00::","synset_id":"couple.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"couple%1:14:01::","call_ids":["senseval3.d002.s109.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"couple%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":430,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01799,"input_uncached_usd":0.00215,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.746908499975689}
{"item_id":"senseval3.d002.s109.t006","gold_sense_keys":["building%1:06:00::"],"candidates":[{"index":1,"sense_key":"building%1:14:00::","synset_id":"building.n.04"},{"index":2,"sense_key":"building%1:06:00::","synset_id":"building.n.01"},{"index":3,"sense_key":"building%1:04:01::","synset_id":"construction.n.07"},{"index":4,"sense_key":"building%1:04:00::","synset_id":"construction.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"building%1:06:00::","call_ids":["senseval3.d002.s109.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"building%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":431,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.003985000000000001,"input_uncached_usd":0.0021550000000000002,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.081497540930286}
{"item_id":"senseval3.d002.s109.t007","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:38:03::","synset_id":"start.v.14"},{"index":2,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":3,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":4,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":5,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":6,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":7,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":8,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":9,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":10,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":11,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":12,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":13,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":14,"sense_key":"start%2:36:01::","synset_id":"start.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"start%2:30:00::","call_ids":["senseval3.d002.s109.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"start%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":874,"cached_input_tokens":0,"output_tokens":2072,"reasoning_output_tokens":2039},"cost":{"currency":"USD","total_usd":0.06653,"input_uncached_usd":0.004370000000000001,"input_cached_usd":0.0,"output_usd":0.06216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":30.344419041997753}
{"item_id":"senseval3.d002.s109.t008","gold_sense_keys":["shake%2:29:00::","shake%2:38:00::","shake%2:38:01::","shake%2:38:02::"],"candidates":[{"index":1,"sense_key":"shake%2:38:02::","synset_id":"rock.v.01"},{"index":2,"sense_key":"shake%2:32:00::","synset_id":"shake.v.09"},{"index":3,"sense_key":"shake%2:38:01::","synset_id":"judder.v.01"},{"index":4,"sense_key":"shake%2:30:00::","synset_id":"shake.v.05"},{"index":5,"sense_key":"shake%2:37:00::","synset_id":"stimulate.v.03"},{"index":6,"sense_key":"shake%2:38:04::","synset_id":"shake.v.07"},{"index":7,"sense_key":"shake%2:30:01::","synset_id":"shake.v.08"},{"index":8,"sense_key":"shake%2:29:00::","synset_id":"shake.v.02"},{"index":9,"sense_key":"shake%2:38:00::","synset_id":"shake.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"shake%2:38:02::","call_ids":["senseval3.d002.s109.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"shake%2:38:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":640,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019010000000000003,"input_uncached_usd":0.0032,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.547531666001305}
{"item_id":"senseval3.d002.s110.t000","gold_sense_keys":["flimsy%5:00:01:weak:00"],"candidates":[{"index":1,"sense_key":"flimsy%3:00:02::","synset_id":"unconvincing.a.01"},{"index":2,"sense_key":"flimsy%5:00:03:insignificant:00","synset_id":"flimsy.s.02"},{"index":3,"sense_key":"flimsy%5:00:01:weak:00","synset_id":"flimsy.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"flimsy%5:00:01:weak:00","call_ids":["senseval3.d002.s110.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"flimsy%5:00:01:weak:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":411,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.003615,"input_uncached_usd":0.002055,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2508312920108438}
{"item_id":"senseval3.d002.s110.t001","gold_sense_keys":["structure%1:06:00::"],"candidates":[{"index":1,"sense_key":"structure%1:07:00::","synset_id":"structure.n.02"},{"index":2,"sense_key":"structure%1:06:00::","synset_id":"structure.n.01"},{"index":3,"sense_key":"structure%1:08:00::","synset_id":"structure.n.04"},{"index":4,"sense_key":"structure%1:09:00::","synset_id":"structure.n.03"},{"index":5,"sense_key":"structure%1:14:00::","synset_id":"social_organization.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"structure%1:06:00::","call_ids":["senseval3.d002.s110.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"structure%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.004275,"input_uncached_usd":0.002625,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.546574124949984}
{"item_id":"senseval3.d002.s110.t002","gold_sense_keys":["build%2:36:00::"],"candidates":[{"index":1,"sense_key":"build%2:30:00::","synset_id":"build_up.v.02"},{"index":2,"sense_key":"build%2:36:04::","synset_id":"build.v.06"},{"index":3,"sense_key":"build%2:36:00::","synset_id":"construct.v.01"},{"index":4,"sense_key":"build%2:36:09::","synset_id":"build.v.03"},{"index":5,"sense_key":"build%2:30:05::","synset_id":"build_up.v.04"},{"index":6,"sense_key":"build%2:41:00::","synset_id":"build.v.05"},{"index":7,"sense_key":"build%2:36:10::","synset_id":"build.v.07"},{"index":8,"sense_key":"build%2:30:02::","synset_id":"build.v.04"},{"index":9,"sense_key":"build%2:31:03::","synset_id":"build.v.08"},{"index":10,"sense_key":"build%2:30:10::","synset_id":"build.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"build%2:36:00::","call_ids":["senseval3.d002.s110.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"build%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":115},"cost":{"currency":"USD","total_usd":0.007170000000000001,"input_uncached_usd":0.0032400000000000003,"input_cached_usd":0.0,"output_usd":0.00393,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.481723582954146}
{"item_id":"senseval3.d002.s110.t003","gold_sense_keys":["support%1:06:01::"],"candidates":[{"index":1,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":2,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":3,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":4,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":9,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":10,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"support%1:06:00::","call_ids":["senseval3.d002.s110.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"support%1:06:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":777,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019725,"input_uncached_usd":0.0038850000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.443436790956184}
{"item_id":"senseval3.d002.s110.t004","gold_sense_keys":["rock%2:38:00::"],"candidates":[{"index":1,"sense_key":"rock%2:38:06::","synset_id":"rock.v.02"},{"index":2,"sense_key":"rock%2:38:00::","synset_id":"rock.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"rock%2:38:00::","call_ids":["senseval3.d002.s110.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"rock%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":345,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.003855,"input_uncached_usd":0.0017250000000000002,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5212409161031246}
{"item_id":"senseval3.d002.s111.t000","gold_sense_keys":["stop%2:42:00::"],"candidates":[{"index":1,"sense_key":"stop%2:30:12::","synset_id":"hold_on.v.02"},{"index":2,"sense_key":"stop%2:35:00::","synset_id":"intercept.v.01"},{"index":3,"sense_key":"stop%2:30:05::","synset_id":"break.v.10"},{"index":4,"sense_key":"stop%2:38:01::","synset_id":"stop.v.05"},{"index":5,"sense_key":"stop%2:38:00::","synset_id":"stop.v.01"},{"index":6,"sense_key":"stop%2:33:00::","synset_id":"check.v.18"},{"index":7,"sense_key":"stop%2:41:00::","synset_id":"stop.v.03"},{"index":8,"sense_key":"stop%2:38:02::","synset_id":"stop.v.04"},{"index":9,"sense_key":"stop%2:42:13::","synset_id":"end.v.01"},{"index":10,"sense_key":"stop%2:35:12::","synset_id":"barricade.v.01"},{"index":11,"sense_key":"stop%2:42:00::","synset_id":"discontinue.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"stop%2:42:00::","call_ids":["senseval3.d002.s111.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"stop%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":791,"cached_input_tokens":0,"output_tokens":341,"reasoning_output_tokens":325},"cost":{"currency":"USD","total_usd":0.014185,"input_uncached_usd":0.003955,"input_cached_usd":0.0,"output_usd":0.01023,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.570359417004511}
{"item_id":"senseval3.d002.s111.t001","gold_sense_keys":["breathe%2:29:00::"],"candidates":[{"index":1,"sense_key":"breathe%2:32:01::","synset_id":"breathe.v.05"},{"index":2,"sense_key":"breathe%2:42:00::","synset_id":"breathe.v.02"},{"index":3,"sense_key":"breathe%2:32:00::","synset_id":"rest.v.02"},{"index":4,"sense_key":"breathe%2:40:00::","synset_id":"breathe.v.03"},{"index":5,"sense_key":"breathe%2:29:00::","synset_id":"breathe.v.01"},{"index":6,"sense_key":"breathe%2:30:03::","synset_id":"breathe.v.08"},{"index":7,"sense_key":"breathe%2:29:01::","synset_id":"emit.v.01"},{"index":8,"sense_key":"breathe%2:32:03::","synset_id":"breathe.v.06"},{"index":9,"sense_key":"breathe%2:42:01::","synset_id":"breathe.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"breathe%2:29:00::","call_ids":["senseval3.d002.s111.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"breathe%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.00525,"input_uncached_usd":0.0030600000000000002,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9443602079991251}
{"item_id":"senseval3.d002.s111.t002","gold_sense_keys":["moment%1:28:01::"],"candidates":[{"index":1,"sense_key":"moment%1:28:02::","synset_id":"here_and_now.n.01"},{"index":2,"sense_key":"moment%1:09:00::","synset_id":"moment.n.06"},{"index":3,"sense_key":"moment%1:19:00::","synset_id":"moment.n.05"},{"index":4,"sense_key":"moment%1:07:00::","synset_id":"consequence.n.03"},{"index":5,"sense_key":"moment%1:28:01::","synset_id":"moment.n.02"},{"index":6,"sense_key":"moment%1:28:00::","synset_id":"moment.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"moment%1:28:01::","call_ids":["senseval3.d002.s111.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"moment%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.00529,"input_uncached_usd":0.0026200000000000004,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8077346669742838}
{"item_id":"senseval3.d002.s111.t003","gold_sense_keys":["keep%2:42:07::"],"candidates":[{"index":1,"sense_key":"keep%2:42:12::","synset_id":"keep.v.16"},{"index":2,"sense_key":"keep%2:32:00::","synset_id":"keep.v.08"},{"index":3,"sense_key":"keep%2:40:01::","synset_id":"keep.v.07"},{"index":4,"sense_key":"keep%2:40:12::","synset_id":"keep.v.19"},{"index":5,"sense_key":"keep%2:40:10::","synset_id":"keep.v.20"},{"index":6,"sense_key":"keep%2:41:05::","synset_id":"observe.v.06"},{"index":7,"sense_key":"keep%2:35:10::","synset_id":"keep.v.21"},{"index":8,"sense_key":"keep%2:31:00::","synset_id":"observe.v.08"},{"index":9,"sense_key":"keep%2:42:02::","synset_id":"keep.v.15"},{"index":10,"sense_key":"keep%2:42:07::","synset_id":"continue.v.01"},{"index":11,"sense_key":"keep%2:42:00::","synset_id":"keep.v.01"},{"index":12,"sense_key":"keep%2:42:01::","synset_id":"keep.v.09"},{"index":13,"sense_key":"keep%2:41:01::","synset_id":"prevent.v.02"},{"index":14,"sense_key":"keep%2:34:00::","synset_id":"sustain.v.04"},{"index":15,"sense_key":"keep%2:40:09::","synset_id":"keep_open.v.01"},{"index":16,"sense_key":"keep%2:41:00::","synset_id":"restrain.v.01"},{"index":17,"sense_key":"keep%2:42:03::","synset_id":"keep.v.12"},{"index":18,"sense_key":"keep%2:41:02::","synset_id":"retain.v.02"},{"index":19,"sense_key":"keep%2:40:00::","synset_id":"keep.v.03"},{"index":20,"sense_key":"keep%2:41:03::","synset_id":"observe.v.09"},{"index":21,"sense_key":"keep%2:30:10::","synset_id":"preserve.v.04"},{"index":22,"sense_key":"keep%2:40:13::","synset_id":"keep.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"keep%2:42:07::","call_ids":["senseval3.d002.s111.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"keep%2:42:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1204,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.00899,"input_uncached_usd":0.00602,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.833228625007905}
{"item_id":"senseval3.d002.s111.t004","gold_sense_keys":["come%2:30:01::"],"candidates":[{"index":1,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":2,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":3,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":4,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":5,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":6,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":7,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":8,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":11,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":12,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":13,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":14,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":15,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":16,"sense_key":"come%2:31:13::","synset_id":"come.v.21"},{"index":17,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":18,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":19,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":20,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":21,"sense_key":"come%2:42:04::","synset_id":"come.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"come%2:30:01::","call_ids":["senseval3.d002.s111.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"come%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1145,"cached_input_tokens":0,"output_tokens":359,"reasoning_output_tokens":343},"cost":{"currency":"USD","total_usd":0.016495,"input_uncached_usd":0.005725,"input_cached_usd":0.0,"output_usd":0.01077,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.477651500026695}
{"item_id":"senseval3.d002.s112.t001","gold_sense_keys":["say%2:32:00::"],"candidates":[{"index":1,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":2,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":3,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":4,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":5,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":6,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":7,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":8,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":9,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":10,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":11,"sense_key":"say%2:32:04::","synset_id":"order.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval3.d002.s112.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":417,"reasoning_output_tokens":401},"cost":{"currency":"USD","total_usd":0.016130000000000002,"input_uncached_usd":0.0036200000000000004,"input_cached_usd":0.0,"output_usd":0.01251,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.44135095900856}
{"item_id":"senseval3.d002.s112.t004","gold_sense_keys":["session%1:04:00::"],"candidates":[{"index":1,"sense_key":"session%1:04:00::","synset_id":"session.n.03"},{"index":2,"sense_key":"session%1:14:00::","synset_id":"seance.n.01"},{"index":3,"sense_key":"session%1:10:00::","synset_id":"session.n.01"},{"index":4,"sense_key":"session%1:28:00::","synset_id":"school_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"session%1:04:00::","call_ids":["senseval3.d002.s112.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"session%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":393,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.003495,"input_uncached_usd":0.001965,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.70054616604466}
{"item_id":"senseval3.d002.s116.t000","gold_sense_keys":["trouble%1:09:00::"],"candidates":[{"index":1,"sense_key":"trouble%1:26:02::","synset_id":"trouble.n.06"},{"index":2,"sense_key":"trouble%1:04:02::","synset_id":"trouble.n.04"},{"index":3,"sense_key":"trouble%1:12:03::","synset_id":"worry.n.02"},{"index":4,"sense_key":"trouble%1:11:00::","synset_id":"trouble.n.03"},{"index":5,"sense_key":"trouble%1:11:01::","synset_id":"fuss.n.02"},{"index":6,"sense_key":"trouble%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"trouble%1:09:00::","call_ids":["senseval3.d002.s116.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"trouble%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":181,"reasoning_output_tokens":165},"cost":{"currency":"USD","total_usd":0.0077800000000000005,"input_uncached_usd":0.00235,"input_cached_usd":0.0,"output_usd":0.00543,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.695736165973358}
{"item_id":"senseval3.d002.s116.t005","gold_sense_keys":["spend%2:42:00::"],"candidates":[{"index":1,"sense_key":"spend%2:42:00::","synset_id":"spend.v.01"},{"index":2,"sense_key":"spend%2:40:00::","synset_id":"spend.v.02"},{"index":3,"sense_key":"spend%2:40:01::","synset_id":"spend.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"spend%2:42:00::","call_ids":["senseval3.d002.s116.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"spend%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":335,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.003385,"input_uncached_usd":0.001675,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.633712959010154}
{"item_id":"senseval3.d002.s116.t007","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"},{"index":2,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"},{"index":3,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"},{"index":4,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"hour%1:28:00::","call_ids":["senseval3.d002.s116.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.0045850000000000005,"input_uncached_usd":0.0019450000000000001,"input_cached_usd":0.0,"output_usd":0.00264,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9219849170185626}
{"item_id":"senseval3.d002.s116.t008","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":2,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":3,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":4,"sense_key":"know%2:31:14::","synset_id":"know.v.11"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":7,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":8,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":9,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":10,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":11,"sense_key":"know%2:31:01::","synset_id":"know.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval3.d002.s116.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":747,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":124},"cost":{"currency":"USD","total_usd":0.007935000000000001,"input_uncached_usd":0.0037350000000000005,"input_cached_usd":0.0,"output_usd":0.0042,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.772968750097789}
{"item_id":"senseval3.d002.s119.t002","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":2,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":3,"sense_key":"start%2:38:03::","synset_id":"start.v.14"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":8,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":9,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":10,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":11,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":12,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":13,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":14,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"start%2:30:00::","call_ids":["senseval3.d002.s119.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"start%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":869,"cached_input_tokens":0,"output_tokens":4672,"reasoning_output_tokens":4601},"cost":{"currency":"USD","total_usd":0.144505,"input_uncached_usd":0.004345000000000001,"input_cached_usd":0.0,"output_usd":0.14016,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":67.59424091596156}
{"item_id":"senseval3.d002.s119.t003","gold_sense_keys":["roll%2:38:10::"],"candidates":[{"index":1,"sense_key":"roll%2:39:00::","synset_id":"roll.v.05"},{"index":2,"sense_key":"roll%2:38:11::","synset_id":"roll.v.14"},{"index":3,"sense_key":"roll%2:40:00::","synset_id":"hustle.v.03"},{"index":4,"sense_key":"roll%2:32:00::","synset_id":"roll.v.15"},{"index":5,"sense_key":"roll%2:42:00::","synset_id":"roll.v.03"},{"index":6,"sense_key":"roll%2:38:01::","synset_id":"wheel.v.03"},{"index":7,"sense_key":"roll%2:30:07::","synset_id":"roll.v.18"},{"index":8,"sense_key":"roll%2:35:00::","synset_id":"wind.v.03"},{"index":9,"sense_key":"roll%2:30:10::","synset_id":"roll.v.08"},{"index":10,"sense_key":"roll%2:38:09::","synset_id":"roll.v.12"},{"index":11,"sense_key":"roll%2:29:06::","synset_id":"roll.v.09"},{"index":12,"sense_key":"roll%2:35:05::","synset_id":"roll_out.v.01"},{"index":13,"sense_key":"roll%2:38:12::","synset_id":"roll.v.13"},{"index":14,"sense_key":"roll%2:35:01::","synset_id":"roll.v.07"},{"index":15,"sense_key":"roll%2:30:06::","synset_id":"roll.v.17"},{"index":16,"sense_key":"roll%2:30:08::","synset_id":"seethe.v.04"},{"index":17,"sense_key":"roll%2:38:00::","synset_id":"roll.v.01"},{"index":18,"sense_key":"roll%2:38:10::","synset_id":"roll.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":18,"chosen_sense_key":"roll%2:38:10::","call_ids":["senseval3.d002.s119.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":18,"predicted_sense_key":"roll%2:38:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":984,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":111},"cost":{"currency":"USD","total_usd":0.008700000000000001,"input_uncached_usd":0.004920000000000001,"input_cached_usd":0.0,"output_usd":0.00378,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8877094999188557}
{"item_id":"senseval3.d002.s119.t004","gold_sense_keys":["ocean%1:17:00::"],"candidates":[{"index":1,"sense_key":"ocean%1:23:00::","synset_id":"ocean.n.02"},{"index":2,"sense_key":"ocean%1:17:00::","synset_id":"ocean.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ocean%1:17:00::","call_ids":["senseval3.d002.s119.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ocean%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":302,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.0043,"input_uncached_usd":0.00151,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.341730957967229}
{"item_id":"senseval3.d002.s119.t005","gold_sense_keys":["wave%1:11:01::"],"candidates":[{"index":1,"sense_key":"wave%1:10:01::","synset_id":"wave.n.05"},{"index":2,"sense_key":"wave%1:11:01::","synset_id":"wave.n.01"},{"index":3,"sense_key":"wave%1:18:00::","synset_id":"wave.n.09"},{"index":4,"sense_key":"wave%1:04:02::","synset_id":"wave.n.02"},{"index":5,"sense_key":"wave%1:08:00::","synset_id":"wave.n.06"},{"index":6,"sense_key":"wave%1:25:00::","synset_id":"wave.n.07"},{"index":7,"sense_key":"wave%1:11:02::","synset_id":"wave.n.04"},{"index":8,"sense_key":"wave%1:19:00::","synset_id":"wave.n.08"},{"index":9,"sense_key":"wave%1:11:00::","synset_id":"wave.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"wave%1:11:01::","call_ids":["senseval3.d002.s119.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"wave%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":147},"cost":{"currency":"USD","total_usd":0.0077150000000000005,"input_uncached_usd":0.0028250000000000003,"input_cached_usd":0.0,"output_usd":0.00489,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6888680000556633}
{"item_id":"senseval3.d002.s120.t000","gold_sense_keys":["run%2:38:00::"],"candidates":[{"index":1,"sense_key":"run%2:42:05::","synset_id":"run.v.30"},{"index":2,"sense_key":"run%2:33:01::","synset_id":"race.v.02"},{"index":3,"sense_key":"run%2:33:06::","synset_id":"campaign.v.01"},{"index":4,"sense_key":"run%2:42:07::","synset_id":"range.v.01"},{"index":5,"sense_key":"run%2:30:03::","synset_id":"run.v.14"},{"index":6,"sense_key":"run%2:35:00::","synset_id":"function.v.01"},{"index":7,"sense_key":"run%2:36:01::","synset_id":"play.v.18"},{"index":8,"sense_key":"run%2:40:06::","synset_id":"run.v.25"},{"index":9,"sense_key":"run%2:42:03::","synset_id":"prevail.v.03"},{"index":10,"sense_key":"run%2:33:02::","synset_id":"hunt.v.01"},{"index":11,"sense_key":"run%2:30:01::","synset_id":"ladder.v.01"},{"index":12,"sense_key":"run%2:41:02::","synset_id":"operate.v.01"},{"index":13,"sense_key":"run%2:38:06::","synset_id":"run.v.33"},{"index":14,"sense_key":"run%2:36:00::","synset_id":"run.v.21"},{"index":15,"sense_key":"run%2:42:06::","synset_id":"run.v.18"},{"index":16,"sense_key":"run%2:42:01::","synset_id":"tend.v.01"},{"index":17,"sense_key":"run%2:38:13::","synset_id":"run.v.34"},{"index":18,"sense_key":"run%2:41:00::","synset_id":"run.v.19"},{"index":19,"sense_key":"run%2:38:04::","synset_id":"scat.v.01"},{"index":20,"sense_key":"run%2:38:08::","synset_id":"run.v.27"},{"index":21,"sense_key":"run%2:36:02::","synset_id":"carry.v.15"},{"index":22,"sense_key":"run%2:42:04::","synset_id":"run.v.05"},{"index":23,"sense_key":"run%2:38:02::","synset_id":"run.v.32"},{"index":24,"sense_key":"run%2:42:02::","synset_id":"run.v.16"},{"index":25,"sense_key":"run%2:38:03::","synset_id":"run.v.28"},{"index":26,"sense_key":"run%2:38:05::","synset_id":"run.v.29"},{"index":27,"sense_key":"run%2:41:01::","synset_id":"run.v.24"},{"index":28,"sense_key":"run%2:38:11::","synset_id":"run.v.11"},{"index":29,"sense_key":"run%2:38:00::","synset_id":"run.v.01"},{"index":30,"sense_key":"run%2:35:04::","synset_id":"run.v.13"},{"index":31,"sense_key":"run%2:42:00::","synset_id":"run.v.03"},{"index":32,"sense_key":"run%2:38:01::","synset_id":"run.v.06"},{"index":33,"sense_key":"run%2:30:07::","synset_id":"run.v.15"},{"index":34,"sense_key":"run%2:30:02::","synset_id":"move.v.13"},{"index":35,"sense_key":"run%2:38:10::","synset_id":"run.v.26"},{"index":36,"sense_key":"run%2:42:08::","synset_id":"run.v.23"},{"index":37,"sense_key":"run%2:41:04::","synset_id":"run.v.31"},{"index":38,"sense_key":"run%2:30:00::","synset_id":"melt.v.01"},{"index":39,"sense_key":"run%2:30:04::","synset_id":"run.v.41"},{"index":40,"sense_key":"run%2:35:01::","synset_id":"guide.v.05"},{"index":41,"sense_key":"run%2:38:09::","synset_id":"ply.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":29,"chosen_sense_key":"run%2:38:00::","call_ids":["senseval3.d002.s120.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":29,"predicted_sense_key":"run%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1952,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":98},"cost":{"currency":"USD","total_usd":0.013150000000000002,"input_uncached_usd":0.009760000000000001,"input_cached_usd":0.0,"output_usd":0.0033900000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.281003708019853}
{"item_id":"senseval3.d002.s120.t001","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:01::","synset_id":"house.n.12"},{"index":2,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":3,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":4,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":5,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":6,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":7,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":8,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":9,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":10,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:14:06::","synset_id":"house.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s120.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":727,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.005285,"input_uncached_usd":0.003635,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":72.5605388750555}
{"item_id":"senseval3.d002.s120.t002","gold_sense_keys":["get%2:35:03::"],"candidates":[{"index":1,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":2,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":3,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":4,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":5,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":6,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":7,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":8,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":9,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":10,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":11,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":12,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":13,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":14,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":15,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":16,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":17,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":18,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":19,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":20,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":21,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":22,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":23,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":24,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":25,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":26,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":27,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":28,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":29,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"},{"index":30,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":31,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":32,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":33,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":34,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":35,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":36,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":25,"chosen_sense_key":"get%2:35:03::","call_ids":["senseval3.d002.s120.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":25,"predicted_sense_key":"get%2:35:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1886,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":110},"cost":{"currency":"USD","total_usd":0.01318,"input_uncached_usd":0.009430000000000001,"input_cached_usd":0.0,"output_usd":0.0037500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9238530000438914}
{"item_id":"senseval3.d002.s120.t003","gold_sense_keys":["next%5:00:00:succeeding:00"],"candidates":[{"index":1,"sense_key":"next%5:00:00:incoming:00","synset_id":"future.s.02"},{"index":2,"sense_key":"next%5:00:00:close:01","synset_id":"adjacent.s.01"},{"index":3,"sense_key":"next%5:00:00:succeeding:00","synset_id":"following.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"next%5:00:00:succeeding:00","call_ids":["senseval3.d002.s120.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"next%5:00:00:succeeding:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":388,"cached_input_tokens":0,"output_tokens":41,"reasoning_output_tokens":26},"cost":{"currency":"USD","total_usd":0.00317,"input_uncached_usd":0.00194,"input_cached_usd":0.0,"output_usd":0.00123,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2214009169256315}
{"item_id":"senseval3.d002.s120.t004","gold_sense_keys":["tremor%1:11:00::"],"candidates":[{"index":1,"sense_key":"tremor%1:11:00::","synset_id":"tremor.n.02"},{"index":2,"sense_key":"tremor%1:26:00::","synset_id":"tremor.n.03"},{"index":3,"sense_key":"tremor%1:04:00::","synset_id":"tremor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tremor%1:11:00::","call_ids":["senseval3.d002.s120.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tremor%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":352,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00353,"input_uncached_usd":0.00176,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0051750000566244}
{"item_id":"senseval3.d002.s120.t005","gold_sense_keys":["throw%2:35:00::"],"candidates":[{"index":1,"sense_key":"throw%2:31:00::","synset_id":"confuse.v.02"},{"index":2,"sense_key":"throw%2:38:00::","synset_id":"throw.v.02"},{"index":3,"sense_key":"throw%2:36:08::","synset_id":"hold.v.03"},{"index":4,"sense_key":"throw%2:35:08::","synset_id":"throw.v.14"},{"index":5,"sense_key":"throw%2:35:00::","synset_id":"throw.v.01"},{"index":6,"sense_key":"throw%2:35:02::","synset_id":"throw.v.04"},{"index":7,"sense_key":"throw%2:32:03::","synset_id":"hurl.v.03"},{"index":8,"sense_key":"throw%2:38:03::","synset_id":"throw.v.08"},{"index":9,"sense_key":"throw%2:37:00::","synset_id":"bewilder.v.02"},{"index":10,"sense_key":"throw%2:35:06::","synset_id":"throw.v.13"},{"index":11,"sense_key":"throw%2:32:00::","synset_id":"give.v.07"},{"index":12,"sense_key":"throw%2:36:00::","synset_id":"throw.v.12"},{"index":13,"sense_key":"throw%2:36:02::","synset_id":"project.v.10"},{"index":14,"sense_key":"throw%2:35:07::","synset_id":"shed.v.01"},{"index":15,"sense_key":"throw%2:35:01::","synset_id":"throw.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"throw%2:35:00::","call_ids":["senseval3.d002.s120.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"throw%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":889,"cached_input_tokens":0,"output_tokens":181,"reasoning_output_tokens":166},"cost":{"currency":"USD","total_usd":0.009875,"input_uncached_usd":0.004445,"input_cached_usd":0.0,"output_usd":0.00543,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5861409589415416}
{"item_id":"senseval3.d002.s120.t006","gold_sense_keys":["air%1:15:00::"],"candidates":[{"index":1,"sense_key":"air%1:19:00::","synset_id":"breeze.n.01"},{"index":2,"sense_key":"air%1:10:02::","synset_id":"air.n.08"},{"index":3,"sense_key":"air%1:04:00::","synset_id":"air_travel.n.01"},{"index":4,"sense_key":"air%1:15:00::","synset_id":"air.n.02"},{"index":5,"sense_key":"air%1:27:00::","synset_id":"air.n.01"},{"index":6,"sense_key":"air%1:15:01::","synset_id":"atmosphere.n.03"},{"index":7,"sense_key":"air%1:27:01::","synset_id":"air.n.06"},{"index":8,"sense_key":"air%1:07:00::","synset_id":"air.n.03"},{"index":9,"sense_key":"air%1:10:01::","synset_id":"tune.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"air%1:15:00::","call_ids":["senseval3.d002.s120.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"air%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.005335,"input_uncached_usd":0.0033850000000000004,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6244019159348682}
{"item_id":"senseval3.d002.s120.t007","gold_sense_keys":["bounce%2:35:03::"],"candidates":[{"index":1,"sense_key":"bounce%2:40:01::","synset_id":"bounce.v.06"},{"index":2,"sense_key":"bounce%2:38:00::","synset_id":"bounce.v.01"},{"index":3,"sense_key":"bounce%2:35:03::","synset_id":"bounce.v.02"},{"index":4,"sense_key":"bounce%2:38:02::","synset_id":"bounce.v.03"},{"index":5,"sense_key":"bounce%2:35:00::","synset_id":"bounce.v.07"},{"index":6,"sense_key":"bounce%2:38:01::","synset_id":"bounce.v.05"},{"index":7,"sense_key":"bounce%2:40:00::","synset_id":"bounce.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"bounce%2:38:02::","call_ids":["senseval3.d002.s120.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"bounce%2:38:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01832,"input_uncached_usd":0.00248,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.57704283297062}
{"item_id":"senseval3.d002.s120.t008","gold_sense_keys":["try%2:41:00::"],"candidates":[{"index":1,"sense_key":"try%2:34:00::","synset_id":"sample.v.01"},{"index":2,"sense_key":"try%2:41:00::","synset_id":"try.v.01"},{"index":3,"sense_key":"try%2:37:00::","synset_id":"try.v.07"},{"index":4,"sense_key":"try%2:41:05::","synset_id":"hear.v.03"},{"index":5,"sense_key":"try%2:41:01::","synset_id":"test.v.01"},{"index":6,"sense_key":"try%2:29:00::","synset_id":"try_on.v.01"},{"index":7,"sense_key":"try%2:37:01::","synset_id":"try.v.06"},{"index":8,"sense_key":"try%2:30:00::","synset_id":"try.v.08"},{"index":9,"sense_key":"try%2:41:06::","synset_id":"judge.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"try%2:41:00::","call_ids":["senseval3.d002.s120.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"try%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":689,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.004795000000000001,"input_uncached_usd":0.003445,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4648729999316856}
{"item_id":"senseval3.d002.s121.t000","gold_sense_keys":["fine%5:00:00:satisfactory:00"],"candidates":[{"index":1,"sense_key":"fine%5:00:00:pure:02","synset_id":"fine.s.05"},{"index":2,"sense_key":"fine%5:00:00:elegant:00","synset_id":"fine.s.04"},{"index":3,"sense_key":"fine%5:00:00:satisfactory:00","synset_id":"all_right.s.01"},{"index":4,"sense_key":"fine%3:00:00::","synset_id":"fine.a.05"},{"index":5,"sense_key":"fine%5:00:00:precise:00","synset_id":"fine.s.02"},{"index":6,"sense_key":"fine%5:00:00:thin:01","synset_id":"fine.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fine%5:00:00:satisfactory:00","call_ids":["senseval3.d002.s121.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fine%5:00:00:satisfactory:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.004165,"input_uncached_usd":0.0025150000000000003,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3515688329935074}
{"item_id":"senseval3.d002.s122.t001","gold_sense_keys":["full%3:00:00::"],"candidates":[{"index":1,"sense_key":"full%5:00:00:complete:00","synset_id":"full.s.02"},{"index":2,"sense_key":"full%5:00:00:high:02","synset_id":"broad.s.04"},{"index":3,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":4,"sense_key":"full%5:00:02:ample:00","synset_id":"wide.s.04"},{"index":5,"sense_key":"full%5:00:00:nourished:00","synset_id":"full.s.03"},{"index":6,"sense_key":"full%3:00:01::","synset_id":"full.a.05"},{"index":7,"sense_key":"full%3:00:00::","synset_id":"full.a.01"},{"index":8,"sense_key":"full%5:00:01:whole:00","synset_id":"entire.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"full%3:00:00::","call_ids":["senseval3.d002.s122.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"full%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.005765,"input_uncached_usd":0.002795,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4257415829924867}
{"item_id":"senseval3.d002.s122.t002","gold_sense_keys":["crystal%1:06:01::"],"candidates":[{"index":1,"sense_key":"crystal%1:06:01::","synset_id":"crystal.n.05"},{"index":2,"sense_key":"crystal%1:27:00::","synset_id":"crystal.n.01"},{"index":3,"sense_key":"crystal%1:27:01::","synset_id":"quartz_glass.n.01"},{"index":4,"sense_key":"crystal%1:06:00::","synset_id":"crystal.n.06"},{"index":5,"sense_key":"crystal%1:17:00::","synset_id":"crystal.n.03"},{"index":6,"sense_key":"crystal%1:06:02::","synset_id":"crystal.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crystal%1:06:01::","call_ids":["senseval3.d002.s122.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crystal%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.004505,"input_uncached_usd":0.0021950000000000003,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3462987079983577}
{"item_id":"senseval3.d002.s123.t000","gold_sense_keys":["book%1:06:00::"],"candidates":[{"index":1,"sense_key":"book%1:14:01::","synset_id":"book.n.06"},{"index":2,"sense_key":"book%1:21:00::","synset_id":"ledger.n.01"},{"index":3,"sense_key":"book%1:10:00::","synset_id":"book.n.01"},{"index":4,"sense_key":"book%1:10:05::","synset_id":"koran.n.01"},{"index":5,"sense_key":"book%1:10:01::","synset_id":"script.n.01"},{"index":6,"sense_key":"book%1:10:03::","synset_id":"book.n.10"},{"index":7,"sense_key":"book%1:14:00::","synset_id":"book.n.07"},{"index":8,"sense_key":"book%1:10:02::","synset_id":"record.n.05"},{"index":9,"sense_key":"book%1:06:02::","synset_id":"book.n.11"},{"index":10,"sense_key":"book%1:06:00::","synset_id":"book.n.02"},{"index":11,"sense_key":"book%1:10:04::","synset_id":"bible.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"book%1:06:00::","call_ids":["senseval3.d002.s123.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"book%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":80},"cost":{"currency":"USD","total_usd":0.006530000000000001,"input_uncached_usd":0.0036500000000000005,"input_cached_usd":0.0,"output_usd":0.00288,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2110445420257747}
{"item_id":"senseval3.d002.s123.t001","gold_sense_keys":["tape%1:06:01::","tape%1:06:04::"],"candidates":[{"index":1,"sense_key":"tape%1:06:03::","synset_id":"tape.n.04"},{"index":2,"sense_key":"tape%1:06:04::","synset_id":"magnetic_tape.n.01"},{"index":3,"sense_key":"tape%1:06:01::","synset_id":"tape.n.02"},{"index":4,"sense_key":"tape%1:15:00::","synset_id":"tape.n.03"},{"index":5,"sense_key":"tape%1:06:00::","synset_id":"tape.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tape%1:06:04::","call_ids":["senseval3.d002.s123.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tape%1:06:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":387,"reasoning_output_tokens":371},"cost":{"currency":"USD","total_usd":0.014105000000000001,"input_uncached_usd":0.0024950000000000003,"input_cached_usd":0.0,"output_usd":0.01161,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.797726665972732}
{"item_id":"senseval3.d002.s123.t002","gold_sense_keys":["room%1:06:00::"],"candidates":[{"index":1,"sense_key":"room%1:26:00::","synset_id":"room.n.03"},{"index":2,"sense_key":"room%1:14:00::","synset_id":"room.n.04"},{"index":3,"sense_key":"room%1:23:00::","synset_id":"room.n.02"},{"index":4,"sense_key":"room%1:06:00::","synset_id":"room.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"room%1:06:00::","call_ids":["senseval3.d002.s123.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"room%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":386,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.0033699999999999997,"input_uncached_usd":0.00193,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7251015841029584}
{"item_id":"senseval3.d002.s124.t000","gold_sense_keys":["thing%1:06:00::"],"candidates":[{"index":1,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":4,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":5,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"},{"index":6,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":7,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":8,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":9,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":10,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":11,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":12,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"thing%1:06:01::","call_ids":["senseval3.d002.s124.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"thing%1:06:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.03478,"input_uncached_usd":0.0034000000000000002,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":16.02231116697658}
{"item_id":"senseval3.d002.s124.t001","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":2,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":3,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":4,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":5,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":6,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":7,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":8,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":9,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":10,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":11,"sense_key":"house%1:06:01::","synset_id":"house.n.12"},{"index":12,"sense_key":"house%1:04:00::","synset_id":"house.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s124.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":757,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.006095,"input_uncached_usd":0.003785,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.316946665989235}
{"item_id":"senseval3.d002.s124.t002","gold_sense_keys":["structure%1:06:00::"],"candidates":[{"index":1,"sense_key":"structure%1:07:00::","synset_id":"structure.n.02"},{"index":2,"sense_key":"structure%1:14:00::","synset_id":"social_organization.n.01"},{"index":3,"sense_key":"structure%1:08:00::","synset_id":"structure.n.04"},{"index":4,"sense_key":"structure%1:09:00::","synset_id":"structure.n.03"},{"index":5,"sense_key":"structure%1:06:00::","synset_id":"structure.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"structure%1:06:00::","call_ids":["senseval3.d002.s124.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"structure%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":147},"cost":{"currency":"USD","total_usd":0.0076100000000000004,"input_uncached_usd":0.00272,"input_cached_usd":0.0,"output_usd":0.00489,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1023108339868486}
{"item_id":"senseval3.d002.s124.t003","gold_sense_keys":["fine%5:00:00:satisfactory:00"],"candidates":[{"index":1,"sense_key":"fine%5:00:00:precise:00","synset_id":"fine.s.02"},{"index":2,"sense_key":"fine%5:00:00:pure:02","synset_id":"fine.s.05"},{"index":3,"sense_key":"fine%5:00:00:thin:01","synset_id":"fine.s.03"},{"index":4,"sense_key":"fine%5:00:00:elegant:00","synset_id":"fine.s.04"},{"index":5,"sense_key":"fine%5:00:00:satisfactory:00","synset_id":"all_right.s.01"},{"index":6,"sense_key":"fine%3:00:00::","synset_id":"fine.a.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"fine%5:00:00:satisfactory:00","call_ids":["senseval3.d002.s124.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"fine%5:00:00:satisfactory:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.004235,"input_uncached_usd":0.0026750000000000003,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.496255333069712}
{"item_id":"senseval3.d002.s125.t000","gold_sense_keys":["stand%2:35:00::","stand%2:42:03::"],"candidates":[{"index":1,"sense_key":"stand%2:31:00::","synset_id":"digest.v.03"},{"index":2,"sense_key":"stand%2:35:10::","synset_id":"stand.v.12"},{"index":3,"sense_key":"stand%2:42:03::","synset_id":"stand.v.03"},{"index":4,"sense_key":"stand%2:42:04::","synset_id":"stand.v.02"},{"index":5,"sense_key":"stand%2:35:01::","synset_id":"stand.v.10"},{"index":6,"sense_key":"stand%2:35:00::","synset_id":"stand.v.01"},{"index":7,"sense_key":"stand%2:42:06::","synset_id":"stand.v.07"},{"index":8,"sense_key":"stand%2:42:02::","synset_id":"stand.v.08"},{"index":9,"sense_key":"stand%2:42:01::","synset_id":"stand.v.09"},{"index":10,"sense_key":"stand%2:31:01::","synset_id":"stand.v.06"},{"index":11,"sense_key":"stand%2:42:00::","synset_id":"stand.v.04"},{"index":12,"sense_key":"stand%2:33:00::","synset_id":"resist.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"stand%2:35:00::","call_ids":["senseval3.d002.s125.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"stand%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":316,"reasoning_output_tokens":300},"cost":{"currency":"USD","total_usd":0.013130000000000001,"input_uncached_usd":0.0036500000000000005,"input_cached_usd":0.0,"output_usd":0.00948,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.6731768330791965}
{"item_id":"senseval3.d002.s125.t002","gold_sense_keys":["wait%2:42:00::"],"candidates":[{"index":1,"sense_key":"wait%2:42:01::","synset_id":"wait.v.02"},{"index":2,"sense_key":"wait%2:41:00::","synset_id":"wait.v.04"},{"index":3,"sense_key":"wait%2:31:00::","synset_id":"expect.v.03"},{"index":4,"sense_key":"wait%2:42:00::","synset_id":"wait.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"wait%2:42:00::","call_ids":["senseval3.d002.s125.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"wait%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":388,"reasoning_output_tokens":373},"cost":{"currency":"USD","total_usd":0.013880000000000002,"input_uncached_usd":0.0022400000000000002,"input_cached_usd":0.0,"output_usd":0.011640000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.868908750009723}
{"item_id":"senseval3.d002.s125.t003","gold_sense_keys":["tremor%1:11:00::"],"candidates":[{"index":1,"sense_key":"tremor%1:26:00::","synset_id":"tremor.n.03"},{"index":2,"sense_key":"tremor%1:04:00::","synset_id":"tremor.n.01"},{"index":3,"sense_key":"tremor%1:11:00::","synset_id":"tremor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"tremor%1:11:00::","call_ids":["senseval3.d002.s125.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"tremor%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00356,"input_uncached_usd":0.00185,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5462742089293897}
{"item_id":"senseval3.d002.s125.t004","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":2,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d002.s125.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":246,"reasoning_output_tokens":230},"cost":{"currency":"USD","total_usd":0.009745,"input_uncached_usd":0.0023650000000000003,"input_cached_usd":0.0,"output_usd":0.00738,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.084515667054802}
{"item_id":"senseval3.d002.s125.t006","gold_sense_keys":["emerge%2:30:00::","emerge%2:30:02::","emerge%2:38:00::"],"candidates":[{"index":1,"sense_key":"emerge%2:30:01::","synset_id":"emerge.v.03"},{"index":2,"sense_key":"emerge%2:38:00::","synset_id":"emerge.v.04"},{"index":3,"sense_key":"emerge%2:42:00::","synset_id":"come_forth.v.02"},{"index":4,"sense_key":"emerge%2:30:00::","synset_id":"emerge.v.01"},{"index":5,"sense_key":"emerge%2:30:02::","synset_id":"issue.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"emerge%2:30:00::","call_ids":["senseval3.d002.s125.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"emerge%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":6222,"reasoning_output_tokens":6141},"cost":{"currency":"USD","total_usd":0.188995,"input_uncached_usd":0.0023350000000000003,"input_cached_usd":0.0,"output_usd":0.18666,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":93.40968574990984}
{"item_id":"senseval3.d002.s125.t007","gold_sense_keys":["ground%1:27:00::"],"candidates":[{"index":1,"sense_key":"ground%1:17:01::","synset_id":"land.n.02"},{"index":2,"sense_key":"ground%1:09:01::","synset_id":"ground.n.08"},{"index":3,"sense_key":"ground%1:06:00::","synset_id":"flat_coat.n.01"},{"index":4,"sense_key":"ground%1:16:00::","synset_id":"reason.n.01"},{"index":5,"sense_key":"ground%1:17:00::","synset_id":"land.n.04"},{"index":6,"sense_key":"ground%1:06:01::","synset_id":"ground.n.10"},{"index":7,"sense_key":"ground%1:06:02::","synset_id":"ground.n.09"},{"index":8,"sense_key":"ground%1:27:00::","synset_id":"earth.n.02"},{"index":9,"sense_key":"ground%1:15:00::","synset_id":"ground.n.05"},{"index":10,"sense_key":"ground%1:24:00::","synset_id":"footing.n.02"},{"index":11,"sense_key":"ground%1:09:00::","synset_id":"background.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ground%1:17:01::","call_ids":["senseval3.d002.s125.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ground%1:17:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":793,"cached_input_tokens":0,"output_tokens":4529,"reasoning_output_tokens":4464},"cost":{"currency":"USD","total_usd":0.13983500000000001,"input_uncached_usd":0.003965000000000001,"input_cached_usd":0.0,"output_usd":0.13587,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":73.73926779103931}
{"item_id":"senseval3.d002.s128.t000","gold_sense_keys":["amazing%5:00:00:surprising:00"],"candidates":[{"index":1,"sense_key":"amazing%5:00:00:surprising:00","synset_id":"amazing.s.01"},{"index":2,"sense_key":"amazing%5:00:00:impressive:00","synset_id":"amazing.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"amazing%5:00:00:surprising:00","call_ids":["senseval3.d002.s128.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"amazing%5:00:00:surprising:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":360,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":94},"cost":{"currency":"USD","total_usd":0.00507,"input_uncached_usd":0.0018000000000000002,"input_cached_usd":0.0,"output_usd":0.00327,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.844103749957867}
{"item_id":"senseval3.d002.s128.t001","gold_sense_keys":["second%1:28:00::"],"candidates":[{"index":1,"sense_key":"second%1:04:00::","synset_id":"second_base.n.02"},{"index":2,"sense_key":"second%1:06:00::","synset_id":"second_gear.n.01"},{"index":3,"sense_key":"second%1:24:00::","synset_id":"second.n.05"},{"index":4,"sense_key":"second%1:10:00::","synset_id":"second.n.08"},{"index":5,"sense_key":"second%1:28:00::","synset_id":"second.n.01"},{"index":6,"sense_key":"second%1:28:02::","synset_id":"moment.n.01"},{"index":7,"sense_key":"second%1:23:00::","synset_id":"second.n.06"},{"index":8,"sense_key":"second%1:28:01::","synset_id":"moment.n.02"},{"index":9,"sense_key":"second%1:18:00::","synset_id":"second.n.07"},{"index":10,"sense_key":"second%1:06:01::","synset_id":"irregular.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"second%1:28:00::","call_ids":["senseval3.d002.s128.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"second%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":1560,"reasoning_output_tokens":1533},"cost":{"currency":"USD","total_usd":0.050095,"input_uncached_usd":0.003295,"input_cached_usd":0.0,"output_usd":0.0468,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":26.976754207978956}
{"item_id":"senseval3.d002.s128.t002","gold_sense_keys":["change%2:30:01::"],"candidates":[{"index":1,"sense_key":"change%2:30:04::","synset_id":"change.v.06"},{"index":2,"sense_key":"change%2:40:00::","synset_id":"exchange.v.01"},{"index":3,"sense_key":"change%2:30:02::","synset_id":"change.v.03"},{"index":4,"sense_key":"change%2:30:06::","synset_id":"change.v.10"},{"index":5,"sense_key":"change%2:30:00::","synset_id":"change.v.02"},{"index":6,"sense_key":"change%2:30:03::","synset_id":"change.v.05"},{"index":7,"sense_key":"change%2:30:01::","synset_id":"change.v.01"},{"index":8,"sense_key":"change%2:38:00::","synset_id":"transfer.v.06"},{"index":9,"sense_key":"change%2:30:05::","synset_id":"switch.v.03"},{"index":10,"sense_key":"change%2:30:08::","synset_id":"deepen.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"change%2:30:01::","call_ids":["senseval3.d002.s128.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"change%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":86},"cost":{"currency":"USD","total_usd":0.006775000000000001,"input_uncached_usd":0.0037150000000000004,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.870237542083487}
{"item_id":"senseval3.d002.s128.t003","gold_sense_keys":["life%1:26:02::"],"candidates":[{"index":1,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":2,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":3,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":4,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":5,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":6,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":7,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"},{"index":8,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":9,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":10,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":11,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":12,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":13,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":14,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"life%1:26:02::","call_ids":["senseval3.d002.s128.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"life%1:26:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":111},"cost":{"currency":"USD","total_usd":0.007835,"input_uncached_usd":0.0040550000000000004,"input_cached_usd":0.0,"output_usd":0.00378,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.345771542051807}
{"item_id":"senseval3.d002.s131.t000","gold_sense_keys":["guess%2:31:00::"],"candidates":[{"index":1,"sense_key":"guess%2:31:00::","synset_id":"think.v.02"},{"index":2,"sense_key":"guess%2:31:01::","synset_id":"estimate.v.01"},{"index":3,"sense_key":"guess%2:32:00::","synset_id":"guess.v.02"},{"index":4,"sense_key":"guess%2:31:02::","synset_id":"guess.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"guess%2:31:00::","call_ids":["senseval3.d002.s131.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"guess%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.005705,"input_uncached_usd":0.0021650000000000003,"input_cached_usd":0.0,"output_usd":0.00354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0611696670530364}
{"item_id":"senseval3.d002.s131.t001","gold_sense_keys":["live%2:42:04::","live%2:42:06::"],"candidates":[{"index":1,"sense_key":"live%2:31:00::","synset_id":"know.v.05"},{"index":2,"sense_key":"live%2:42:06::","synset_id":"live.v.02"},{"index":3,"sense_key":"live%2:42:04::","synset_id":"survive.v.01"},{"index":4,"sense_key":"live%2:42:07::","synset_id":"live.v.07"},{"index":5,"sense_key":"live%2:42:00::","synset_id":"be.v.11"},{"index":6,"sense_key":"live%2:42:08::","synset_id":"populate.v.01"},{"index":7,"sense_key":"live%2:42:01::","synset_id":"exist.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"live%2:42:06::","call_ids":["senseval3.d002.s131.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"live%2:42:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":564,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.0058200000000000005,"input_uncached_usd":0.00282,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.467047167010605}
{"item_id":"senseval3.d002.s131.t002","gold_sense_keys":["wait%2:31:00::"],"candidates":[{"index":1,"sense_key":"wait%2:31:00::","synset_id":"expect.v.03"},{"index":2,"sense_key":"wait%2:42:00::","synset_id":"wait.v.01"},{"index":3,"sense_key":"wait%2:41:00::","synset_id":"wait.v.04"},{"index":4,"sense_key":"wait%2:42:01::","synset_id":"wait.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wait%2:31:00::","call_ids":["senseval3.d002.s131.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wait%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":2479,"reasoning_output_tokens":2438},"cost":{"currency":"USD","total_usd":0.076315,"input_uncached_usd":0.0019450000000000001,"input_cached_usd":0.0,"output_usd":0.07437,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":38.839189291000366}
{"item_id":"senseval3.d002.s132.t000","gold_sense_keys":["hard%3:00:06::"],"candidates":[{"index":1,"sense_key":"hard%3:00:06::","synset_id":"difficult.a.01"},{"index":2,"sense_key":"hard%3:00:00::","synset_id":"unvoiced.a.01"},{"index":3,"sense_key":"hard%3:00:05::","synset_id":"hard.a.07"},{"index":4,"sense_key":"hard%5:00:00:stale:00","synset_id":"hard.s.06"},{"index":5,"sense_key":"hard%3:00:03::","synset_id":"hard.a.08"},{"index":6,"sense_key":"hard%3:00:02::","synset_id":"hard.a.02"},{"index":7,"sense_key":"hard%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":8,"sense_key":"hard%5:00:00:indulgent:00","synset_id":"intemperate.s.01"},{"index":9,"sense_key":"hard%3:00:01::","synset_id":"hard.a.03"},{"index":10,"sense_key":"hard%5:00:00:alcoholic:00","synset_id":"hard.s.04"},{"index":11,"sense_key":"hard%5:00:01:effortful:00","synset_id":"arduous.s.01"},{"index":12,"sense_key":"hard%5:00:00:bad:00","synset_id":"hard.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hard%3:00:06::","call_ids":["senseval3.d002.s132.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hard%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":744,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.005730000000000001,"input_uncached_usd":0.00372,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5516957909567282}
{"item_id":"senseval3.d002.s132.t001","gold_sense_keys":["accept%2:31:00::"],"candidates":[{"index":1,"sense_key":"accept%2:40:12::","synset_id":"take.v.20"},{"index":2,"sense_key":"accept%2:40:07::","synset_id":"accept.v.09"},{"index":3,"sense_key":"accept%2:40:00::","synset_id":"accept.v.02"},{"index":4,"sense_key":"accept%2:31:01::","synset_id":"accept.v.04"},{"index":5,"sense_key":"accept%2:40:01::","synset_id":"bear.v.06"},{"index":6,"sense_key":"accept%2:40:03::","synset_id":"accept.v.05"},{"index":7,"sense_key":"accept%2:31:08::","synset_id":"accept.v.11"},{"index":8,"sense_key":"accept%2:42:00::","synset_id":"accept.v.08"},{"index":9,"sense_key":"accept%2:32:00::","synset_id":"accept.v.03"},{"index":10,"sense_key":"accept%2:31:00::","synset_id":"accept.v.01"},{"index":11,"sense_key":"accept%2:31:03::","synset_id":"accept.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"accept%2:31:00::","call_ids":["senseval3.d002.s132.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"accept%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019345,"input_uncached_usd":0.0035050000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.062931458000094}
{"item_id":"senseval3.d002.s132.t002","gold_sense_keys":["take%2:40:06::"],"candidates":[{"index":1,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":2,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":3,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":4,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":5,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":6,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":7,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":8,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":9,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":10,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":11,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":12,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":13,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":14,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":15,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":16,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":19,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":20,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":21,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":22,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":23,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":24,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":25,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":26,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":27,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":28,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"},{"index":29,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":32,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":33,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":34,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":35,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":36,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":37,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":38,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":39,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":40,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":41,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":42,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":41,"chosen_sense_key":"take%2:40:06::","call_ids":["senseval3.d002.s132.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":41,"predicted_sense_key":"take%2:40:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2080,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.013400000000000002,"input_uncached_usd":0.010400000000000001,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.35387879202608}
{"item_id":"senseval3.d002.s132.t003","gold_sense_keys":["second%1:28:00::"],"candidates":[{"index":1,"sense_key":"second%1:10:00::","synset_id":"second.n.08"},{"index":2,"sense_key":"second%1:28:02::","synset_id":"moment.n.01"},{"index":3,"sense_key":"second%1:28:00::","synset_id":"second.n.01"},{"index":4,"sense_key":"second%1:18:00::","synset_id":"second.n.07"},{"index":5,"sense_key":"second%1:28:01::","synset_id":"moment.n.02"},{"index":6,"sense_key":"second%1:06:01::","synset_id":"irregular.n.02"},{"index":7,"sense_key":"second%1:06:00::","synset_id":"second_gear.n.01"},{"index":8,"sense_key":"second%1:24:00::","synset_id":"second.n.05"},{"index":9,"sense_key":"second%1:23:00::","synset_id":"second.n.06"},{"index":10,"sense_key":"second%1:04:00::","synset_id":"second_base.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"second%1:28:00::","call_ids":["senseval3.d002.s132.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"second%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.00477,"input_uncached_usd":0.0031500000000000005,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6822104579769075}
{"item_id":"senseval3.d002.s133.t000","gold_sense_keys":["wonder%2:31:00::"],"candidates":[{"index":1,"sense_key":"wonder%2:32:12::","synset_id":"wonder.v.03"},{"index":2,"sense_key":"wonder%2:32:01::","synset_id":"wonder.v.02"},{"index":3,"sense_key":"wonder%2:31:00::","synset_id":"wonder.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"wonder%2:31:00::","call_ids":["senseval3.d002.s133.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"wonder%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":345,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":124},"cost":{"currency":"USD","total_usd":0.005895,"input_uncached_usd":0.0017250000000000002,"input_cached_usd":0.0,"output_usd":0.00417,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.812229000031948}
{"item_id":"senseval3.d002.s133.t001","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"},{"index":2,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"},{"index":3,"sense_key":"able%3:00:00::","synset_id":"able.a.01"},{"index":4,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"able%3:00:00::","call_ids":["senseval3.d002.s133.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"able%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":405,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":109},"cost":{"currency":"USD","total_usd":0.005775000000000001,"input_uncached_usd":0.0020250000000000003,"input_cached_usd":0.0,"output_usd":0.0037500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.029458750039339}
{"item_id":"senseval3.d002.s133.t002","gold_sense_keys":["relax%2:29:00::"],"candidates":[{"index":1,"sense_key":"relax%2:41:02::","synset_id":"relax.v.05"},{"index":2,"sense_key":"relax%2:29:01::","synset_id":"relax.v.04"},{"index":3,"sense_key":"relax%2:30:05::","synset_id":"slack.v.04"},{"index":4,"sense_key":"relax%2:41:01::","synset_id":"relax.v.06"},{"index":5,"sense_key":"relax%2:30:01::","synset_id":"relax.v.02"},{"index":6,"sense_key":"relax%2:41:00::","synset_id":"relax.v.07"},{"index":7,"sense_key":"relax%2:30:02::","synset_id":"loosen.v.07"},{"index":8,"sense_key":"relax%2:29:00::","synset_id":"relax.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"relax%2:29:00::","call_ids":["senseval3.d002.s133.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"relax%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.00563,"input_uncached_usd":0.00302,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7996450000209734}
{"item_id":"senseval3.d002.s136.t000","gold_sense_keys":["flesh%1:08:01::"],"candidates":[{"index":1,"sense_key":"flesh%1:08:01::","synset_id":"human_body.n.01"},{"index":2,"sense_key":"flesh%1:20:00::","synset_id":"pulp.n.02"},{"index":3,"sense_key":"flesh%1:08:02::","synset_id":"flesh.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"flesh%1:08:01::","call_ids":["senseval3.d002.s136.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"flesh%1:08:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":337,"cached_input_tokens":0,"output_tokens":352,"reasoning_output_tokens":336},"cost":{"currency":"USD","total_usd":0.012245,"input_uncached_usd":0.001685,"input_cached_usd":0.0,"output_usd":0.01056,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.839457625057548}
{"item_id":"senseval3.d002.s136.t001","gold_sense_keys":["go%2:30:04::"],"candidates":[{"index":1,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":2,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":3,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":4,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":5,"sense_key":"go%2:42:08::","synset_id":"go.v.25"},{"index":6,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":7,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":8,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":9,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"},{"index":10,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":11,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":12,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":13,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"},{"index":14,"sense_key":"go%2:30:08::","synset_id":"go.v.09"},{"index":15,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":16,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":17,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":18,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":19,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":20,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":21,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":22,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":23,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":24,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":25,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"},{"index":26,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":27,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":28,"sense_key":"go%2:38:01::","synset_id":"go.v.03"},{"index":29,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":30,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":22,"chosen_sense_key":"go%2:30:04::","call_ids":["senseval3.d002.s136.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":22,"predicted_sense_key":"go%2:30:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1617,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":116},"cost":{"currency":"USD","total_usd":0.012045,"input_uncached_usd":0.008085,"input_cached_usd":0.0,"output_usd":0.00396,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.47128183406312}
{"item_id":"senseval3.d002.s136.t003","gold_sense_keys":["flight%1:04:01::"],"candidates":[{"index":1,"sense_key":"flight%1:19:00::","synset_id":"trajectory.n.01"},{"index":2,"sense_key":"flight%1:06:00::","synset_id":"flight.n.03"},{"index":3,"sense_key":"flight%1:14:01::","synset_id":"flight.n.01"},{"index":4,"sense_key":"flight%1:04:00::","synset_id":"flight.n.02"},{"index":5,"sense_key":"flight%1:09:00::","synset_id":"flight.n.06"},{"index":6,"sense_key":"flight%1:14:02::","synset_id":"flight.n.08"},{"index":7,"sense_key":"flight%1:14:00::","synset_id":"flight.n.05"},{"index":8,"sense_key":"flight%1:04:02::","synset_id":"flight.n.09"},{"index":9,"sense_key":"flight%1:04:01::","synset_id":"escape.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"flight%1:04:01::","call_ids":["senseval3.d002.s136.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"flight%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.004855,"input_uncached_usd":0.0025450000000000004,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6447665840387344}
{"item_id":"senseval3.d002.s136.t004","gold_sense_keys":["fight%1:04:02::"],"candidates":[{"index":1,"sense_key":"fight%1:10:00::","synset_id":"fight.n.04"},{"index":2,"sense_key":"fight%1:04:01::","synset_id":"battle.n.01"},{"index":3,"sense_key":"fight%1:04:02::","synset_id":"fight.n.02"},{"index":4,"sense_key":"fight%1:04:03::","synset_id":"fight.n.05"},{"index":5,"sense_key":"fight%1:07:00::","synset_id":"competitiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fight%1:04:02::","call_ids":["senseval3.d002.s136.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fight%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":166,"reasoning_output_tokens":151},"cost":{"currency":"USD","total_usd":0.007165,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.00498,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4606605409644544}
{"item_id":"senseval3.d002.s137.t000","gold_sense_keys":["nausea%1:26:00::"],"candidates":[{"index":1,"sense_key":"nausea%1:26:00::","synset_id":"nausea.n.01"},{"index":2,"sense_key":"nausea%1:12:00::","synset_id":"nausea.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nausea%1:26:00::","call_ids":["senseval3.d002.s137.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nausea%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":261,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":113},"cost":{"currency":"USD","total_usd":0.005175000000000001,"input_uncached_usd":0.0013050000000000002,"input_cached_usd":0.0,"output_usd":0.00387,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5416020000120625}
{"item_id":"senseval3.d002.s137.t001","gold_sense_keys":["seem%2:39:01::"],"candidates":[{"index":1,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":2,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"},{"index":3,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":4,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval3.d002.s137.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"seem%2:39:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":519,"reasoning_output_tokens":504},"cost":{"currency":"USD","total_usd":0.017515,"input_uncached_usd":0.0019450000000000001,"input_cached_usd":0.0,"output_usd":0.01557,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.644587374990806}
{"item_id":"senseval3.d002.s137.t002","gold_sense_keys":["commonplace%5:00:02:ordinary:00"],"candidates":[{"index":1,"sense_key":"commonplace%5:00:02:ordinary:00","synset_id":"commonplace.s.01"},{"index":2,"sense_key":"commonplace%5:00:00:unexciting:00","synset_id":"commonplace.s.02"},{"index":3,"sense_key":"commonplace%5:00:00:unoriginal:00","synset_id":"banal.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"commonplace%5:00:02:ordinary:00","call_ids":["senseval3.d002.s137.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"commonplace%5:00:02:ordinary:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":378,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.00408,"input_uncached_usd":0.0018900000000000002,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0338860420743003}
{"item_id":"senseval3.d002.s137.t003","gold_sense_keys":["symptom%1:26:00::"],"candidates":[{"index":1,"sense_key":"symptom%1:10:00::","synset_id":"symptom.n.02"},{"index":2,"sense_key":"symptom%1:26:00::","synset_id":"symptom.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"symptom%1:26:00::","call_ids":["senseval3.d002.s137.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"symptom%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":277,"cached_input_tokens":0,"output_tokens":413,"reasoning_output_tokens":397},"cost":{"currency":"USD","total_usd":0.013775,"input_uncached_usd":0.0013850000000000002,"input_cached_usd":0.0,"output_usd":0.01239,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.802897959016263}
{"item_id":"senseval3.d002.s139.t000","gold_sense_keys":["walk%2:38:00::"],"candidates":[{"index":1,"sense_key":"walk%2:38:02::","synset_id":"walk.v.08"},{"index":2,"sense_key":"walk%2:41:01::","synset_id":"walk.v.07"},{"index":3,"sense_key":"walk%2:38:03::","synset_id":"walk.v.04"},{"index":4,"sense_key":"walk%2:38:00::","synset_id":"walk.v.01"},{"index":5,"sense_key":"walk%2:33:01::","synset_id":"walk.v.05"},{"index":6,"sense_key":"walk%2:38:05::","synset_id":"walk.v.09"},{"index":7,"sense_key":"walk%2:33:00::","synset_id":"walk.v.03"},{"index":8,"sense_key":"walk%2:38:01::","synset_id":"walk.v.02"},{"index":9,"sense_key":"walk%2:41:00::","synset_id":"walk.v.06"},{"index":10,"sense_key":"walk%2:38:04::","synset_id":"walk.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"walk%2:38:03::","call_ids":["senseval3.d002.s139.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"walk%2:38:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.03431,"input_uncached_usd":0.0029300000000000003,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":16.843269334058277}
{"item_id":"senseval3.d002.s139.t002","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":2,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":3,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":4,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":5,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":6,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"minute%1:28:00::","call_ids":["senseval3.d002.s139.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.0049,"input_uncached_usd":0.0024400000000000003,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7764832079410553}
{"item_id":"senseval3.d002.s139.t003","gold_sense_keys":["morning%1:28:00::"],"candidates":[{"index":1,"sense_key":"morning%1:28:01::","synset_id":"dawn.n.01"},{"index":2,"sense_key":"morning%1:10:00::","synset_id":"good_morning.n.01"},{"index":3,"sense_key":"morning%1:11:00::","synset_id":"dawn.n.02"},{"index":4,"sense_key":"morning%1:28:00::","synset_id":"morning.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"morning%1:28:00::","call_ids":["senseval3.d002.s139.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"morning%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":412,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.00455,"input_uncached_usd":0.00206,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9113394170999527}
{"item_id":"senseval3.d002.s140.t000","gold_sense_keys":["next%5:00:00:close:01"],"candidates":[{"index":1,"sense_key":"next%5:00:00:close:01","synset_id":"adjacent.s.01"},{"index":2,"sense_key":"next%5:00:00:succeeding:00","synset_id":"following.s.02"},{"index":3,"sense_key":"next%5:00:00:incoming:00","synset_id":"future.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"next%5:00:00:close:01","call_ids":["senseval3.d002.s140.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"next%5:00:00:close:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":376,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.0035900000000000003,"input_uncached_usd":0.0018800000000000002,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9842606249731034}
{"item_id":"senseval3.d002.s140.t004","gold_sense_keys":["blue%5:00:00:chromatic:00"],"candidates":[{"index":1,"sense_key":"blue%5:00:00:nonindulgent:00","synset_id":"blue.s.07"},{"index":2,"sense_key":"blue%5:00:00:dejected:00","synset_id":"gloomy.s.02"},{"index":3,"sense_key":"blue%5:00:00:sexy:00","synset_id":"blue.s.05"},{"index":4,"sense_key":"blue%5:00:00:noble:02","synset_id":"aristocratic.s.01"},{"index":5,"sense_key":"blue%5:00:00:chromatic:00","synset_id":"blue.s.01"},{"index":6,"sense_key":"blue%5:00:02:dirty:02","synset_id":"blasphemous.s.02"},{"index":7,"sense_key":"blue%5:00:00:depressing:00","synset_id":"blue.s.08"},{"index":8,"sense_key":"blue%5:00:00:northern:02","synset_id":"blue.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"blue%5:00:00:chromatic:00","call_ids":["senseval3.d002.s140.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"blue%5:00:00:chromatic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.0048400000000000006,"input_uncached_usd":0.0034900000000000005,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8639696659520268}
{"item_id":"senseval3.d002.s140.t006","gold_sense_keys":["sit_up%2:38:01::"],"candidates":[{"index":1,"sense_key":"sit_up%2:29:00::","synset_id":"stay_up.v.01"},{"index":2,"sense_key":"sit_up%2:38:01::","synset_id":"sit_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sit_up%2:38:01::","call_ids":["senseval3.d002.s140.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sit_up%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":327,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.003405,"input_uncached_usd":0.0016350000000000002,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.429547416977584}
{"item_id":"senseval3.d002.s140.t007","gold_sense_keys":["say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":3,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":4,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":5,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":6,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":7,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":8,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":9,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":10,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":11,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"say%2:32:15::","call_ids":["senseval3.d002.s140.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"say%2:32:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":129},"cost":{"currency":"USD","total_usd":0.007965,"input_uncached_usd":0.003615,"input_cached_usd":0.0,"output_usd":0.00435,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.64434054098092}
{"item_id":"senseval3.d002.s140.t009","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"},{"index":2,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":3,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":4,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval3.d002.s140.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.004625000000000001,"input_uncached_usd":0.0021950000000000003,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7998432089807466}
{"item_id":"senseval3.d002.s140.t010","gold_sense_keys":["smile%2:29:00::"],"candidates":[{"index":1,"sense_key":"smile%2:29:00::","synset_id":"smile.v.01"},{"index":2,"sense_key":"smile%2:32:00::","synset_id":"smile.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"smile%2:29:00::","call_ids":["senseval3.d002.s140.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"smile%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":301,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.0036049999999999997,"input_uncached_usd":0.001505,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.843919749953784}
{"item_id":"senseval3.d002.s140.t011","gold_sense_keys":["say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:09::","synset_id":"say.v.11"},{"index":2,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":3,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":4,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":5,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":6,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":9,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":10,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":11,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"say%2:32:15::","call_ids":["senseval3.d002.s140.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"say%2:32:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.034995,"input_uncached_usd":0.003615,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.889690999989398}
{"item_id":"senseval3.d002.s140.t012","gold_sense_keys":["great%5:00:00:good:01"],"candidates":[{"index":1,"sense_key":"great%5:00:00:pregnant:00","synset_id":"big.s.12"},{"index":2,"sense_key":"great%5:00:00:uppercase:00","synset_id":"capital.s.03"},{"index":3,"sense_key":"great%5:00:01:important:00","synset_id":"great.s.02"},{"index":4,"sense_key":"great%5:00:00:good:01","synset_id":"bang-up.s.01"},{"index":5,"sense_key":"great%5:00:01:large:00","synset_id":"great.s.01"},{"index":6,"sense_key":"great%5:00:01:extraordinary:00","synset_id":"great.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"great%5:00:00:good:01","call_ids":["senseval3.d002.s140.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"great%5:00:00:good:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00448,"input_uncached_usd":0.00265,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.709225666942075}
{"item_id":"senseval3.d002.s140.t013","gold_sense_keys":["alive%3:00:01::"],"candidates":[{"index":1,"sense_key":"alive%5:00:00:active:05","synset_id":"alive.s.05"},{"index":2,"sense_key":"alive%5:00:00:sensitive:02","synset_id":"alive.s.02"},{"index":3,"sense_key":"alive%5:00:00:existent:00","synset_id":"active.s.03"},{"index":4,"sense_key":"alive%3:00:04::","synset_id":"animated.a.01"},{"index":5,"sense_key":"alive%3:00:01::","synset_id":"alive.a.01"},{"index":6,"sense_key":"alive%5:00:00:lively:00","synset_id":"alive.s.01"},{"index":7,"sense_key":"alive%5:00:00:aware:00","synset_id":"alert.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"alive%3:00:01::","call_ids":["senseval3.d002.s140.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"alive%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":94},"cost":{"currency":"USD","total_usd":0.00592,"input_uncached_usd":0.0026200000000000004,"input_cached_usd":0.0,"output_usd":0.0033,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7730095830047503}
{"item_id":"senseval3.d002.s141.t000","gold_sense_keys":["agree%2:32:00::"],"candidates":[{"index":1,"sense_key":"agree%2:32:00::","synset_id":"agree.v.01"},{"index":2,"sense_key":"agree%2:32:01::","synset_id":"agree.v.02"},{"index":3,"sense_key":"agree%2:42:03::","synset_id":"agree.v.05"},{"index":4,"sense_key":"agree%2:42:00::","synset_id":"match.v.01"},{"index":5,"sense_key":"agree%2:32:04::","synset_id":"agree.v.07"},{"index":6,"sense_key":"agree%2:40:02::","synset_id":"agree.v.06"},{"index":7,"sense_key":"agree%2:42:01::","synset_id":"harmonize.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"agree%2:32:00::","call_ids":["senseval3.d002.s141.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"agree%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":81},"cost":{"currency":"USD","total_usd":0.00566,"input_uncached_usd":0.0027800000000000004,"input_cached_usd":0.0,"output_usd":0.00288,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.087584207998589}
{"item_id":"semeval2013.d000.s000.t000","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:09:00::","synset_id":"group.n.03"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:03:00::","synset_id":"group.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"group%1:03:00::","call_ids":["semeval2013.d000.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":336,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.00345,"input_uncached_usd":0.00168,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.56996945803985}
{"item_id":"semeval2013.d000.s000.t001","gold_sense_keys":["plan%1:09:00::"],"candidates":[{"index":1,"sense_key":"plan%1:06:00::","synset_id":"plan.n.03"},{"index":2,"sense_key":"plan%1:09:01::","synset_id":"design.n.02"},{"index":3,"sense_key":"plan%1:09:00::","synset_id":"plan.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"plan%1:09:00::","call_ids":["semeval2013.d000.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"plan%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":355,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.003095,"input_uncached_usd":0.001775,"input_cached_usd":0.0,"output_usd":0.00132,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6312433329876512}
{"item_id":"semeval2013.d000.s000.t002","gold_sense_keys":["emission%1:04:00::","emission%1:27:00::"],"candidates":[{"index":1,"sense_key":"emission%1:27:00::","synset_id":"discharge.n.03"},{"index":2,"sense_key":"emission%1:11:00::","synset_id":"emission.n.05"},{"index":3,"sense_key":"emission%1:04:00::","synset_id":"emission.n.01"},{"index":4,"sense_key":"emission%1:22:01::","synset_id":"discharge.n.04"},{"index":5,"sense_key":"emission%1:22:00::","synset_id":"emission.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"emission%1:27:00::","call_ids":["semeval2013.d000.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"emission%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":381,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":115},"cost":{"currency":"USD","total_usd":0.005805,"input_uncached_usd":0.0019050000000000002,"input_cached_usd":0.0,"output_usd":0.0039000000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9713018339825794}
{"item_id":"semeval2013.d000.s001.t000","gold_sense_keys":["climate%1:26:00::"],"candidates":[{"index":1,"sense_key":"climate%1:26:01::","synset_id":"climate.n.02"},{"index":2,"sense_key":"climate%1:26:00::","synset_id":"climate.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"climate%1:26:00::","call_ids":["semeval2013.d000.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"climate%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":349,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.004085,"input_uncached_usd":0.0017450000000000002,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.613839917001314}
{"item_id":"semeval2013.d000.s001.t001","gold_sense_keys":["conference%1:10:00::","conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"},{"index":2,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"},{"index":3,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conference%1:14:00::","call_ids":["semeval2013.d000.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":351,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.004035,"input_uncached_usd":0.001755,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.932197332964279}
{"item_id":"semeval2013.d000.s001.t004","gold_sense_keys":["focus%1:09:00::"],"candidates":[{"index":1,"sense_key":"focus%1:07:03::","synset_id":"focus.n.03"},{"index":2,"sense_key":"focus%1:26:01::","synset_id":"stress.n.03"},{"index":3,"sense_key":"focus%1:07:01::","synset_id":"focus.n.02"},{"index":4,"sense_key":"focus%1:26:00::","synset_id":"focus.n.04"},{"index":5,"sense_key":"focus%1:09:00::","synset_id":"focus.n.01"},{"index":6,"sense_key":"focus%1:15:00::","synset_id":"focus.n.07"},{"index":7,"sense_key":"focus%1:19:00::","synset_id":"focus.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"focus%1:09:00::","call_ids":["semeval2013.d000.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"focus%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":300,"reasoning_output_tokens":284},"cost":{"currency":"USD","total_usd":0.011675,"input_uncached_usd":0.0026750000000000003,"input_cached_usd":0.0,"output_usd":0.009000000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.236612166976556}
{"item_id":"semeval2013.d000.s001.t007","gold_sense_keys":["document%1:10:00::"],"candidates":[{"index":1,"sense_key":"document%1:10:01::","synset_id":"text_file.n.01"},{"index":2,"sense_key":"document%1:10:00::","synset_id":"document.n.01"},{"index":3,"sense_key":"document%1:21:00::","synset_id":"document.n.03"},{"index":4,"sense_key":"document%1:06:00::","synset_id":"document.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"document%1:10:00::","call_ids":["semeval2013.d000.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"document%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":376,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.0037400000000000003,"input_uncached_usd":0.0018800000000000002,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5699983750237152}
{"item_id":"semeval2013.d000.s001.t008","gold_sense_keys":["reduction%1:04:00::"],"candidates":[{"index":1,"sense_key":"reduction%1:04:02::","synset_id":"reduction.n.03"},{"index":2,"sense_key":"reduction%1:22:00::","synset_id":"reduction.n.02"},{"index":3,"sense_key":"reduction%1:04:00::","synset_id":"decrease.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"reduction%1:04:00::","call_ids":["semeval2013.d000.s001.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"reduction%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":363,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.0032250000000000004,"input_uncached_usd":0.0018150000000000002,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2794429579516873}
{"item_id":"semeval2013.d000.s001.t009","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":2,"sense_key":"year%1:14:00::","synset_id":"class.n.06"},{"index":3,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":4,"sense_key":"year%1:28:01::","synset_id":"year.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d000.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.0039900000000000005,"input_uncached_usd":0.00222,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6369163329945877}
{"item_id":"semeval2013.d000.s001.t010","gold_sense_keys":["burden%1:09:01::"],"candidates":[{"index":1,"sense_key":"burden%1:09:01::","synset_id":"burden.n.01"},{"index":2,"sense_key":"burden%1:06:01::","synset_id":"load.n.01"},{"index":3,"sense_key":"burden%1:10:00::","synset_id":"effect.n.04"},{"index":4,"sense_key":"burden%1:09:00::","synset_id":"burden.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"burden%1:09:01::","call_ids":["semeval2013.d000.s001.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"burden%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":401,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.004135,"input_uncached_usd":0.0020050000000000003,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.728870457969606}
{"item_id":"semeval2013.d000.s001.t011","gold_sense_keys":["term%1:28:00::"],"candidates":[{"index":1,"sense_key":"term%1:10:00::","synset_id":"term.n.01"},{"index":2,"sense_key":"term%1:28:00::","synset_id":"term.n.02"},{"index":3,"sense_key":"term%1:06:00::","synset_id":"terminus.n.03"},{"index":4,"sense_key":"term%1:09:00::","synset_id":"term.n.04"},{"index":5,"sense_key":"term%1:28:01::","synset_id":"term.n.06"},{"index":6,"sense_key":"term%1:10:02::","synset_id":"condition.n.07"},{"index":7,"sense_key":"term%1:10:01::","synset_id":"term.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"term%1:28:00::","call_ids":["semeval2013.d000.s001.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"term%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.0042450000000000005,"input_uncached_usd":0.002745,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2728810000699013}
{"item_id":"semeval2013.d000.s002.t000","gold_sense_keys":["text%1:10:00::","text%1:10:01::"],"candidates":[{"index":1,"sense_key":"text%1:10:03::","synset_id":"text.n.02"},{"index":2,"sense_key":"text%1:10:01::","synset_id":"text.n.04"},{"index":3,"sense_key":"text%1:10:02::","synset_id":"textbook.n.01"},{"index":4,"sense_key":"text%1:10:00::","synset_id":"text.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"text%1:10:00::","call_ids":["semeval2013.d000.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"text%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":237,"reasoning_output_tokens":222},"cost":{"currency":"USD","total_usd":0.00949,"input_uncached_usd":0.00238,"input_cached_usd":0.0,"output_usd":0.00711,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.111909957951866}
{"item_id":"semeval2013.d000.s002.t001","gold_sense_keys":["basis%1:09:00::"],"candidates":[{"index":1,"sense_key":"basis%1:24:01::","synset_id":"basis.n.03"},{"index":2,"sense_key":"basis%1:24:00::","synset_id":"footing.n.02"},{"index":3,"sense_key":"basis%1:09:00::","synset_id":"basis.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"basis%1:09:00::","call_ids":["semeval2013.d000.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"basis%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":423,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017955,"input_uncached_usd":0.002115,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.779318375047296}
{"item_id":"semeval2013.d000.s002.t002","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":4,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"},{"index":5,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":6,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":7,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":8,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":9,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d000.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":674,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.00493,"input_uncached_usd":0.00337,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.737276791012846}
{"item_id":"semeval2013.d000.s002.t004","gold_sense_keys":["obstacle%1:09:00::"],"candidates":[{"index":1,"sense_key":"obstacle%1:06:00::","synset_id":"obstacle.n.02"},{"index":2,"sense_key":"obstacle%1:09:00::","synset_id":"obstacle.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"obstacle%1:09:00::","call_ids":["semeval2013.d000.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"obstacle%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":380,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.0034600000000000004,"input_uncached_usd":0.0019000000000000002,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5189702920615673}
{"item_id":"semeval2013.d000.s002.t005","gold_sense_keys":["path%1:04:00::"],"candidates":[{"index":1,"sense_key":"path%1:17:00::","synset_id":"path.n.04"},{"index":2,"sense_key":"path%1:15:00::","synset_id":"path.n.03"},{"index":3,"sense_key":"path%1:06:00::","synset_id":"path.n.02"},{"index":4,"sense_key":"path%1:04:00::","synset_id":"way.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"path%1:04:00::","call_ids":["semeval2013.d000.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"path%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":423,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.004665000000000001,"input_uncached_usd":0.002115,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.113326375023462}
{"item_id":"semeval2013.d000.s003.t000","gold_sense_keys":["advance%1:11:01::"],"candidates":[{"index":1,"sense_key":"advance%1:10:00::","synset_id":"overture.n.03"},{"index":2,"sense_key":"advance%1:04:01::","synset_id":"advance.n.06"},{"index":3,"sense_key":"advance%1:21:00::","synset_id":"advance.n.05"},{"index":4,"sense_key":"advance%1:11:01::","synset_id":"improvement.n.01"},{"index":5,"sense_key":"advance%1:04:00::","synset_id":"progress.n.02"},{"index":6,"sense_key":"advance%1:11:00::","synset_id":"progress.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"advance%1:04:00::","call_ids":["semeval2013.d000.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"advance%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018555000000000002,"input_uncached_usd":0.0027150000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.912550208042376}
{"item_id":"semeval2013.d000.s003.t001","gold_sense_keys":["negotiation%1:10:00::"],"candidates":[{"index":1,"sense_key":"negotiation%1:04:00::","synset_id":"negotiation.n.02"},{"index":2,"sense_key":"negotiation%1:10:00::","synset_id":"negotiation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"negotiation%1:10:00::","call_ids":["semeval2013.d000.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"negotiation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017855,"input_uncached_usd":0.0020150000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.884495625039563}
{"item_id":"semeval2013.d000.s003.t002","gold_sense_keys":["time%1:28:05::"],"candidates":[{"index":1,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":2,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":5,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":6,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":7,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":8,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":9,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"},{"index":10,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"time%1:28:05::","call_ids":["semeval2013.d000.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"time%1:28:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.005475,"input_uncached_usd":0.003615,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8049526669783518}
{"item_id":"semeval2013.d000.s003.t004","gold_sense_keys":["leader%1:18:00::"],"candidates":[{"index":1,"sense_key":"leader%1:18:00::","synset_id":"leader.n.01"},{"index":2,"sense_key":"leader%1:06:00::","synset_id":"drawing_card.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"leader%1:18:00::","call_ids":["semeval2013.d000.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"leader%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":382,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.0035,"input_uncached_usd":0.0019100000000000002,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.587837417027913}
{"item_id":"semeval2013.d000.s003.t006","gold_sense_keys":["week%1:28:02::"],"candidates":[{"index":1,"sense_key":"week%1:28:02::","synset_id":"week.n.03"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:00::","synset_id":"week.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"week%1:28:02::","call_ids":["semeval2013.d000.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"week%1:28:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":165,"reasoning_output_tokens":149},"cost":{"currency":"USD","total_usd":0.007085000000000001,"input_uncached_usd":0.002135,"input_cached_usd":0.0,"output_usd":0.00495,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.830669166985899}
{"item_id":"semeval2013.d000.s004.t001","gold_sense_keys":["text%1:10:00::","text%1:10:01::"],"candidates":[{"index":1,"sense_key":"text%1:10:02::","synset_id":"textbook.n.01"},{"index":2,"sense_key":"text%1:10:00::","synset_id":"text.n.01"},{"index":3,"sense_key":"text%1:10:03::","synset_id":"text.n.02"},{"index":4,"sense_key":"text%1:10:01::","synset_id":"text.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"text%1:10:00::","call_ids":["semeval2013.d000.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"text%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":94},"cost":{"currency":"USD","total_usd":0.0061,"input_uncached_usd":0.0028000000000000004,"input_cached_usd":0.0,"output_usd":0.0033,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3990591670153663}
{"item_id":"semeval2013.d000.s004.t002","gold_sense_keys":["money%1:21:00::"],"candidates":[{"index":1,"sense_key":"money%1:21:02::","synset_id":"money.n.02"},{"index":2,"sense_key":"money%1:21:01::","synset_id":"money.n.03"},{"index":3,"sense_key":"money%1:21:00::","synset_id":"money.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"money%1:21:00::","call_ids":["semeval2013.d000.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"money%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.0059050000000000005,"input_uncached_usd":0.0023650000000000003,"input_cached_usd":0.0,"output_usd":0.00354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4102860828861594}
{"item_id":"semeval2013.d000.s004.t003","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":4,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":5,"sense_key":"country%1:14:00::","synset_id":"state.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":124},"cost":{"currency":"USD","total_usd":0.007260000000000001,"input_uncached_usd":0.0030600000000000002,"input_cached_usd":0.0,"output_usd":0.0042,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9749594579916447}
{"item_id":"semeval2013.d000.s004.t004","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["semeval2013.d000.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.00507,"input_uncached_usd":0.00231,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1115951670799404}
{"item_id":"semeval2013.d000.s004.t006","gold_sense_keys":["term%1:28:00::"],"candidates":[{"index":1,"sense_key":"term%1:28:01::","synset_id":"term.n.06"},{"index":2,"sense_key":"term%1:10:01::","synset_id":"term.n.05"},{"index":3,"sense_key":"term%1:10:02::","synset_id":"condition.n.07"},{"index":4,"sense_key":"term%1:10:00::","synset_id":"term.n.01"},{"index":5,"sense_key":"term%1:09:00::","synset_id":"term.n.04"},{"index":6,"sense_key":"term%1:06:00::","synset_id":"terminus.n.03"},{"index":7,"sense_key":"term%1:28:00::","synset_id":"term.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"term%1:28:00::","call_ids":["semeval2013.d000.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"term%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.004905,"input_uncached_usd":0.0033150000000000002,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.640093959053047}
{"item_id":"semeval2013.d000.s005.t000","gold_sense_keys":["range%1:14:00::"],"candidates":[{"index":1,"sense_key":"range%1:09:00::","synset_id":"compass.n.03"},{"index":2,"sense_key":"range%1:06:01::","synset_id":"stove.n.01"},{"index":3,"sense_key":"range%1:15:02::","synset_id":"range.n.03"},{"index":4,"sense_key":"range%1:17:00::","synset_id":"range.n.04"},{"index":5,"sense_key":"range%1:14:01::","synset_id":"image.n.07"},{"index":6,"sense_key":"range%1:15:01::","synset_id":"range.n.02"},{"index":7,"sense_key":"range%1:06:00::","synset_id":"range.n.05"},{"index":8,"sense_key":"range%1:14:00::","synset_id":"range.n.06"},{"index":9,"sense_key":"range%1:07:00::","synset_id":"scope.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"range%1:14:00::","call_ids":["semeval2013.d000.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"range%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":872,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.00604,"input_uncached_usd":0.00436,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":49.204283541068435}
{"item_id":"semeval2013.d000.s005.t001","gold_sense_keys":["option%1:09:00::"],"candidates":[{"index":1,"sense_key":"option%1:21:00::","synset_id":"option.n.01"},{"index":2,"sense_key":"option%1:04:00::","synset_id":"choice.n.02"},{"index":3,"sense_key":"option%1:09:00::","synset_id":"option.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"option%1:09:00::","call_ids":["semeval2013.d000.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"option%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.0047550000000000005,"input_uncached_usd":0.0026550000000000002,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0787317500216886}
{"item_id":"semeval2013.d000.s005.t002","gold_sense_keys":["question%1:10:00::","question%1:10:01::"],"candidates":[{"index":1,"sense_key":"question%1:10:03::","synset_id":"question.n.03"},{"index":2,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"},{"index":3,"sense_key":"question%1:10:00::","synset_id":"question.n.01"},{"index":4,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":5,"sense_key":"question%1:10:04::","synset_id":"question.n.06"},{"index":6,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"question%1:10:01::","call_ids":["semeval2013.d000.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"question%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.00592,"input_uncached_usd":0.0034000000000000002,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4647499159909785}
{"item_id":"semeval2013.d000.s005.t003","gold_sense_keys":["upper_limit%1:23:00::"],"candidates":[{"index":1,"sense_key":"upper_limit%1:23:00::","synset_id":"maximum.n.01"},{"index":2,"sense_key":"upper_limit%1:15:00::","synset_id":"upper_limit.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"upper_limit%1:23:00::","call_ids":["semeval2013.d000.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"upper_limit%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.00409,"input_uncached_usd":0.00223,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4561164999613538}
{"item_id":"semeval2013.d000.s005.t004","gold_sense_keys":["temperature%1:07:00::"],"candidates":[{"index":1,"sense_key":"temperature%1:07:00::","synset_id":"temperature.n.01"},{"index":2,"sense_key":"temperature%1:09:00::","synset_id":"temperature.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"temperature%1:07:00::","call_ids":["semeval2013.d000.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"temperature%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.00429,"input_uncached_usd":0.0022500000000000003,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6504177919123322}
{"item_id":"semeval2013.d000.s005.t005","gold_sense_keys":["rise%1:11:00::"],"candidates":[{"index":1,"sense_key":"rise%1:07:01::","synset_id":"upgrade.n.04"},{"index":2,"sense_key":"rise%1:11:00::","synset_id":"rise.n.01"},{"index":3,"sense_key":"rise%1:04:00::","synset_id":"rise.n.02"},{"index":4,"sense_key":"rise%1:07:02::","synset_id":"raise.n.01"},{"index":5,"sense_key":"rise%1:04:01::","synset_id":"advance.n.06"},{"index":6,"sense_key":"rise%1:07:03::","synset_id":"rise.n.09"},{"index":7,"sense_key":"rise%1:11:01::","synset_id":"lift.n.04"},{"index":8,"sense_key":"rise%1:11:02::","synset_id":"rise.n.04"},{"index":9,"sense_key":"rise%1:11:03::","synset_id":"emanation.n.03"},{"index":10,"sense_key":"rise%1:17:00::","synset_id":"ascent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"rise%1:11:00::","call_ids":["semeval2013.d000.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"rise%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":796,"cached_input_tokens":0,"output_tokens":353,"reasoning_output_tokens":337},"cost":{"currency":"USD","total_usd":0.01457,"input_uncached_usd":0.00398,"input_cached_usd":0.0,"output_usd":0.01059,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.007444333983585}
{"item_id":"semeval2013.d000.s006.t000","gold_sense_keys":["lot%1:23:00::"],"candidates":[{"index":1,"sense_key":"lot%1:14:00::","synset_id":"set.n.05"},{"index":2,"sense_key":"lot%1:23:00::","synset_id":"batch.n.02"},{"index":3,"sense_key":"lot%1:14:01::","synset_id":"bunch.n.03"},{"index":4,"sense_key":"lot%1:18:00::","synset_id":"lot.n.07"},{"index":5,"sense_key":"lot%1:15:00::","synset_id":"lot.n.02"},{"index":6,"sense_key":"lot%1:06:00::","synset_id":"draw.n.04"},{"index":7,"sense_key":"lot%1:26:00::","synset_id":"fortune.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"lot%1:23:00::","call_ids":["semeval2013.d000.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"lot%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.005260000000000001,"input_uncached_usd":0.0037900000000000004,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7630289590451866}
{"item_id":"semeval2013.d000.s006.t001","gold_sense_keys":["flexibility%1:07:01::"],"candidates":[{"index":1,"sense_key":"flexibility%1:07:01::","synset_id":"flexibility.n.02"},{"index":2,"sense_key":"flexibility%1:07:02::","synset_id":"flexibility.n.01"},{"index":3,"sense_key":"flexibility%1:07:03::","synset_id":"tractability.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"flexibility%1:07:01::","call_ids":["semeval2013.d000.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"flexibility%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.00438,"input_uncached_usd":0.00267,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.583069291897118}
{"item_id":"semeval2013.d000.s006.t002","gold_sense_keys":["process%1:04:00::"],"candidates":[{"index":1,"sense_key":"process%1:09:00::","synset_id":"process.n.02"},{"index":2,"sense_key":"process%1:08:00::","synset_id":"process.n.05"},{"index":3,"sense_key":"process%1:04:00::","synset_id":"procedure.n.01"},{"index":4,"sense_key":"process%1:09:01::","synset_id":"process.n.04"},{"index":5,"sense_key":"process%1:10:00::","synset_id":"summons.n.03"},{"index":6,"sense_key":"process%1:03:00::","synset_id":"process.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"process%1:04:00::","call_ids":["semeval2013.d000.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"process%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":103},"cost":{"currency":"USD","total_usd":0.007195,"input_uncached_usd":0.003625,"input_cached_usd":0.0,"output_usd":0.0035700000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6221395829925314}
{"item_id":"semeval2013.d000.s006.t003","gold_sense_keys":["washington%1:15:01::"],"candidates":[{"index":1,"sense_key":"washington%1:18:00::","synset_id":"washington.n.04"},{"index":2,"sense_key":"washington%1:18:01::","synset_id":"washington.n.05"},{"index":3,"sense_key":"washington%1:15:01::","synset_id":"washington.n.01"},{"index":4,"sense_key":"washington%1:15:00::","synset_id":"washington.n.02"},{"index":5,"sense_key":"washington%1:14:00::","synset_id":"capital.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"washington%1:15:01::","call_ids":["semeval2013.d000.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"washington%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":393,"reasoning_output_tokens":377},"cost":{"currency":"USD","total_usd":0.015130000000000001,"input_uncached_usd":0.00334,"input_cached_usd":0.0,"output_usd":0.01179,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.917335791978985}
{"item_id":"semeval2013.d000.s006.t004","gold_sense_keys":["representative%1:18:00::","representative%1:18:02::"],"candidates":[{"index":1,"sense_key":"representative%1:09:00::","synset_id":"example.n.01"},{"index":2,"sense_key":"representative%1:18:02::","synset_id":"spokesperson.n.01"},{"index":3,"sense_key":"representative%1:18:00::","synset_id":"representative.n.01"},{"index":4,"sense_key":"representative%1:18:01::","synset_id":"congressman.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"representative%1:18:02::","call_ids":["semeval2013.d000.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"representative%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":171,"reasoning_output_tokens":155},"cost":{"currency":"USD","total_usd":0.008100000000000001,"input_uncached_usd":0.0029700000000000004,"input_cached_usd":0.0,"output_usd":0.00513,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5294530419632792}
{"item_id":"semeval2013.d000.s007.t000","gold_sense_keys":["document%1:10:00::"],"candidates":[{"index":1,"sense_key":"document%1:10:00::","synset_id":"document.n.01"},{"index":2,"sense_key":"document%1:21:00::","synset_id":"document.n.03"},{"index":3,"sense_key":"document%1:06:00::","synset_id":"document.n.02"},{"index":4,"sense_key":"document%1:10:01::","synset_id":"text_file.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"document%1:10:00::","call_ids":["semeval2013.d000.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"document%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.0053,"input_uncached_usd":0.00272,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.633864000090398}
{"item_id":"semeval2013.d000.s007.t001","gold_sense_keys":["negotiation%1:04:00::"],"candidates":[{"index":1,"sense_key":"negotiation%1:10:00::","synset_id":"negotiation.n.01"},{"index":2,"sense_key":"negotiation%1:04:00::","synset_id":"negotiation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"negotiation%1:10:00::","call_ids":["semeval2013.d000.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"negotiation%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018349999999999998,"input_uncached_usd":0.00251,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.821346917073242}
{"item_id":"semeval2013.d000.s007.t002","gold_sense_keys":["text%1:10:00::","text%1:10:01::"],"candidates":[{"index":1,"sense_key":"text%1:10:03::","synset_id":"text.n.02"},{"index":2,"sense_key":"text%1:10:01::","synset_id":"text.n.04"},{"index":3,"sense_key":"text%1:10:00::","synset_id":"text.n.01"},{"index":4,"sense_key":"text%1:10:02::","synset_id":"textbook.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"text%1:10:00::","call_ids":["semeval2013.d000.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"text%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":140},"cost":{"currency":"USD","total_usd":0.007755,"input_uncached_usd":0.003075,"input_cached_usd":0.0,"output_usd":0.00468,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.160159334074706}
{"item_id":"semeval2013.d000.s007.t003","gold_sense_keys":["climate%1:26:00::"],"candidates":[{"index":1,"sense_key":"climate%1:26:01::","synset_id":"climate.n.02"},{"index":2,"sense_key":"climate%1:26:00::","synset_id":"climate.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"climate%1:26:00::","call_ids":["semeval2013.d000.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"climate%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.0058000000000000005,"input_uncached_usd":0.0025900000000000003,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.485215208027512}
{"item_id":"semeval2013.d000.s007.t004","gold_sense_keys":["official%1:18:01::"],"candidates":[{"index":1,"sense_key":"official%1:18:00::","synset_id":"official.n.02"},{"index":2,"sense_key":"official%1:18:01::","synset_id":"official.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"official%1:18:01::","call_ids":["semeval2013.d000.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"official%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.004370000000000001,"input_uncached_usd":0.0024500000000000004,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8418587499763817}
{"item_id":"semeval2013.d000.s007.t005","gold_sense_keys":["picture%1:26:00::"],"candidates":[{"index":1,"sense_key":"picture%1:10:03::","synset_id":"word_picture.n.01"},{"index":2,"sense_key":"picture%1:06:00::","synset_id":"picture.n.01"},{"index":3,"sense_key":"picture%1:10:01::","synset_id":"video.n.01"},{"index":4,"sense_key":"picture%1:06:02::","synset_id":"photograph.n.01"},{"index":5,"sense_key":"picture%1:09:01::","synset_id":"picture.n.09"},{"index":6,"sense_key":"picture%1:06:01::","synset_id":"painting.n.01"},{"index":7,"sense_key":"picture%1:10:00::","synset_id":"picture.n.05"},{"index":8,"sense_key":"picture%1:26:00::","synset_id":"picture.n.04"},{"index":9,"sense_key":"picture%1:09:00::","synset_id":"mental_picture.n.01"},{"index":10,"sense_key":"picture%1:10:02::","synset_id":"movie.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"picture%1:26:00::","call_ids":["semeval2013.d000.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"picture%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":953,"cached_input_tokens":0,"output_tokens":422,"reasoning_output_tokens":406},"cost":{"currency":"USD","total_usd":0.017425,"input_uncached_usd":0.004765,"input_cached_usd":0.0,"output_usd":0.012660000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.017047291970812}
{"item_id":"semeval2013.d000.s008.t000","gold_sense_keys":["outline%1:09:00::"],"candidates":[{"index":1,"sense_key":"outline%1:15:00::","synset_id":"outline.n.01"},{"index":2,"sense_key":"outline%1:10:00::","synset_id":"outline.n.02"},{"index":3,"sense_key":"outline%1:09:00::","synset_id":"outline.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"outline%1:09:00::","call_ids":["semeval2013.d000.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"outline%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":98},"cost":{"currency":"USD","total_usd":0.005925,"input_uncached_usd":0.0025050000000000003,"input_cached_usd":0.0,"output_usd":0.0034200000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8060911250067875}
{"item_id":"semeval2013.d000.s008.t001","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":5,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":6,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":7,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":8,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d000.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":799,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.005615,"input_uncached_usd":0.003995,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.722149624954909}
{"item_id":"semeval2013.d000.s008.t002","gold_sense_keys":["emission%1:04:00::","emission%1:27:00::"],"candidates":[{"index":1,"sense_key":"emission%1:11:00::","synset_id":"emission.n.05"},{"index":2,"sense_key":"emission%1:22:01::","synset_id":"discharge.n.04"},{"index":3,"sense_key":"emission%1:04:00::","synset_id":"emission.n.01"},{"index":4,"sense_key":"emission%1:22:00::","synset_id":"emission.n.03"},{"index":5,"sense_key":"emission%1:27:00::","synset_id":"discharge.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"emission%1:27:00::","call_ids":["semeval2013.d000.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"emission%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":288,"reasoning_output_tokens":272},"cost":{"currency":"USD","total_usd":0.011485,"input_uncached_usd":0.0028450000000000003,"input_cached_usd":0.0,"output_usd":0.00864,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.32099154102616}
{"item_id":"semeval2013.d000.s008.t004","gold_sense_keys":["level%1:07:00::"],"candidates":[{"index":1,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"},{"index":2,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":3,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":4,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":5,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":6,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":7,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":8,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"level%1:07:00::","call_ids":["semeval2013.d000.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"level%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.006175,"input_uncached_usd":0.0038350000000000003,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.663025917019695}
{"item_id":"semeval2013.d000.s008.t006","gold_sense_keys":["period%1:28:00::"],"candidates":[{"index":1,"sense_key":"period%1:10:00::","synset_id":"period.n.07"},{"index":2,"sense_key":"period%1:28:03::","synset_id":"period.n.04"},{"index":3,"sense_key":"period%1:22:00::","synset_id":"menstruation.n.01"},{"index":4,"sense_key":"period%1:28:02::","synset_id":"period.n.02"},{"index":5,"sense_key":"period%1:28:00::","synset_id":"time_period.n.01"},{"index":6,"sense_key":"period%1:28:07::","synset_id":"period.n.03"},{"index":7,"sense_key":"period%1:28:04::","synset_id":"period.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"period%1:28:00::","call_ids":["semeval2013.d000.s008.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"period%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":757,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.005615,"input_uncached_usd":0.003785,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6645047080237418}
{"item_id":"semeval2013.d000.s009.t000","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":5,"sense_key":"country%1:15:01::","synset_id":"country.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":676,"cached_input_tokens":0,"output_tokens":209,"reasoning_output_tokens":193},"cost":{"currency":"USD","total_usd":0.00965,"input_uncached_usd":0.00338,"input_cached_usd":0.0,"output_usd":0.00627,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.668498042039573}
{"item_id":"semeval2013.d000.s009.t001","gold_sense_keys":["emission%1:04:00::","emission%1:27:00::"],"candidates":[{"index":1,"sense_key":"emission%1:27:00::","synset_id":"discharge.n.03"},{"index":2,"sense_key":"emission%1:22:01::","synset_id":"discharge.n.04"},{"index":3,"sense_key":"emission%1:11:00::","synset_id":"emission.n.05"},{"index":4,"sense_key":"emission%1:22:00::","synset_id":"emission.n.03"},{"index":5,"sense_key":"emission%1:04:00::","synset_id":"emission.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"emission%1:27:00::","call_ids":["semeval2013.d000.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"emission%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":590,"cached_input_tokens":0,"output_tokens":468,"reasoning_output_tokens":452},"cost":{"currency":"USD","total_usd":0.01699,"input_uncached_usd":0.0029500000000000004,"input_cached_usd":0.0,"output_usd":0.01404,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.65295324998442}
{"item_id":"semeval2013.d000.s010.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:14:00::","synset_id":"class.n.06"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:28:02::","synset_id":"year.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d000.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00479,"input_uncached_usd":0.00302,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.541290375054814}
{"item_id":"semeval2013.d000.s010.t005","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":2,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":3,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d000.s010.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":104},"cost":{"currency":"USD","total_usd":0.00662,"input_uncached_usd":0.00302,"input_cached_usd":0.0,"output_usd":0.0036,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.883136999909766}
{"item_id":"semeval2013.d000.s010.t006","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":2,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":3,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":4,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"country%1:15:01::","synset_id":"country.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s010.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":657,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019125,"input_uncached_usd":0.003285,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.525718750082888}
{"item_id":"semeval2013.d000.s010.t007","gold_sense_keys":["impact%1:19:00::"],"candidates":[{"index":1,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":2,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"},{"index":3,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":4,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"impact%1:19:00::","call_ids":["semeval2013.d000.s010.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"impact%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.00455,"input_uncached_usd":0.00284,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.448733957950026}
{"item_id":"semeval2013.d000.s010.t009","gold_sense_keys":["flood%1:19:00::"],"candidates":[{"index":1,"sense_key":"flood%1:11:00::","synset_id":"flood_tide.n.02"},{"index":2,"sense_key":"flood%1:19:00::","synset_id":"flood.n.01"},{"index":3,"sense_key":"flood%1:23:00::","synset_id":"flood.n.02"},{"index":4,"sense_key":"flood%1:04:00::","synset_id":"flood.n.05"},{"index":5,"sense_key":"flood%1:06:00::","synset_id":"flood.n.03"},{"index":6,"sense_key":"flood%1:04:01::","synset_id":"flood.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"flood%1:19:00::","call_ids":["semeval2013.d000.s010.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"flood%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":673,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.005195,"input_uncached_usd":0.0033650000000000004,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.935662833042443}
{"item_id":"semeval2013.d000.s010.t010","gold_sense_keys":["drought%1:26:00::"],"candidates":[{"index":1,"sense_key":"drought%1:28:00::","synset_id":"drought.n.02"},{"index":2,"sense_key":"drought%1:26:00::","synset_id":"drought.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"drought%1:26:00::","call_ids":["semeval2013.d000.s010.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"drought%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.0039050000000000005,"input_uncached_usd":0.002465,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4209109169896692}
{"item_id":"semeval2013.d000.s011.t001","gold_sense_keys":["sort%1:09:00::"],"candidates":[{"index":1,"sense_key":"sort%1:22:00::","synset_id":"sort.n.04"},{"index":2,"sense_key":"sort%1:07:00::","synset_id":"sort.n.02"},{"index":3,"sense_key":"sort%1:18:00::","synset_id":"sort.n.03"},{"index":4,"sense_key":"sort%1:09:00::","synset_id":"kind.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"sort%1:09:00::","call_ids":["semeval2013.d000.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"sort%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.005745,"input_uncached_usd":0.002925,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1913812500424683}
{"item_id":"semeval2013.d000.s011.t003","gold_sense_keys":["week%1:28:00::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:02::","synset_id":"week.n.03"},{"index":3,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"week%1:28:00::","call_ids":["semeval2013.d000.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"week%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00448,"input_uncached_usd":0.00256,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5829864160623401}
{"item_id":"semeval2013.d000.s012.t001","gold_sense_keys":["climate%1:26:00::"],"candidates":[{"index":1,"sense_key":"climate%1:26:01::","synset_id":"climate.n.02"},{"index":2,"sense_key":"climate%1:26:00::","synset_id":"climate.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"climate%1:26:00::","call_ids":["semeval2013.d000.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"climate%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.005495,"input_uncached_usd":0.0025250000000000003,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0368693749187514}
{"item_id":"semeval2013.d000.s012.t002","gold_sense_keys":["envoy%1:18:00::","envoy%1:18:01::"],"candidates":[{"index":1,"sense_key":"envoy%1:18:01::","synset_id":"emissary.n.01"},{"index":2,"sense_key":"envoy%1:18:00::","synset_id":"envoy.n.01"},{"index":3,"sense_key":"envoy%1:10:00::","synset_id":"envoy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"envoy%1:18:01::","call_ids":["semeval2013.d000.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"envoy%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":396,"reasoning_output_tokens":380},"cost":{"currency":"USD","total_usd":0.01442,"input_uncached_usd":0.00254,"input_cached_usd":0.0,"output_usd":0.01188,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.103037041961215}
{"item_id":"semeval2013.d000.s012.t004","gold_sense_keys":["cut%1:04:05::"],"candidates":[{"index":1,"sense_key":"cut%1:10:04::","synset_id":"cut.n.08"},{"index":2,"sense_key":"cut%1:10:00::","synset_id":"cut.n.02"},{"index":3,"sense_key":"cut%1:13:00::","synset_id":"cut.n.06"},{"index":4,"sense_key":"cut%1:04:09::","synset_id":"cut.n.15"},{"index":5,"sense_key":"cut%1:06:00::","synset_id":"cut.n.03"},{"index":6,"sense_key":"cut%1:26:00::","synset_id":"cut.n.05"},{"index":7,"sense_key":"cut%1:04:02::","synset_id":"cut.n.16"},{"index":8,"sense_key":"cut%1:04:06::","synset_id":"cut.n.20"},{"index":9,"sense_key":"cut%1:04:07::","synset_id":"baseball_swing.n.01"},{"index":10,"sense_key":"cut%1:04:00::","synset_id":"cut.n.18"},{"index":11,"sense_key":"cut%1:10:02::","synset_id":"stinger.n.02"},{"index":12,"sense_key":"cut%1:09:00::","synset_id":"cut.n.10"},{"index":13,"sense_key":"cut%1:04:01::","synset_id":"cut.n.17"},{"index":14,"sense_key":"cut%1:04:03::","synset_id":"snub.n.02"},{"index":15,"sense_key":"cut%1:26:01::","synset_id":"cut.n.04"},{"index":16,"sense_key":"cut%1:21:00::","synset_id":"cut.n.01"},{"index":17,"sense_key":"cut%1:10:01::","synset_id":"deletion.n.03"},{"index":18,"sense_key":"cut%1:04:08::","synset_id":"cut.n.14"},{"index":19,"sense_key":"cut%1:04:05::","synset_id":"cut.n.19"},{"index":20,"sense_key":"cut%1:06:01::","synset_id":"cut.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"cut%1:04:05::","call_ids":["semeval2013.d000.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"cut%1:04:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1200,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00792,"input_uncached_usd":0.006,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.683736416976899}
{"item_id":"semeval2013.d000.s012.t005","gold_sense_keys":["emission%1:04:00::","emission%1:27:00::"],"candidates":[{"index":1,"sense_key":"emission%1:11:00::","synset_id":"emission.n.05"},{"index":2,"sense_key":"emission%1:22:00::","synset_id":"emission.n.03"},{"index":3,"sense_key":"emission%1:27:00::","synset_id":"discharge.n.03"},{"index":4,"sense_key":"emission%1:04:00::","synset_id":"emission.n.01"},{"index":5,"sense_key":"emission%1:22:01::","synset_id":"discharge.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"emission%1:27:00::","call_ids":["semeval2013.d000.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"emission%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":137},"cost":{"currency":"USD","total_usd":0.0074,"input_uncached_usd":0.00284,"input_cached_usd":0.0,"output_usd":0.00456,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2749815420247614}
{"item_id":"semeval2013.d000.s012.t006","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:09:00::","synset_id":"one.n.02"},{"index":2,"sense_key":"one%1:23:00::","synset_id":"one.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"one%1:09:00::","call_ids":["semeval2013.d000.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.00573,"input_uncached_usd":0.00252,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.514841584023088}
{"item_id":"semeval2013.d000.s013.t000","gold_sense_keys":["move%1:04:00::"],"candidates":[{"index":1,"sense_key":"move%1:04:02::","synset_id":"move.n.05"},{"index":2,"sense_key":"move%1:04:00::","synset_id":"move.n.01"},{"index":3,"sense_key":"move%1:04:01::","synset_id":"move.n.02"},{"index":4,"sense_key":"move%1:04:04::","synset_id":"motion.n.06"},{"index":5,"sense_key":"move%1:04:03::","synset_id":"motion.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"move%1:04:00::","call_ids":["semeval2013.d000.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"move%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":635,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.005095000000000001,"input_uncached_usd":0.0031750000000000003,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.782696457928978}
{"item_id":"semeval2013.d000.s013.t002","gold_sense_keys":["china%1:15:00::"],"candidates":[{"index":1,"sense_key":"china%1:06:01::","synset_id":"chinaware.n.01"},{"index":2,"sense_key":"china%1:06:00::","synset_id":"china.n.02"},{"index":3,"sense_key":"china%1:15:00::","synset_id":"china.n.01"},{"index":4,"sense_key":"china%1:15:01::","synset_id":"taiwan.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"china%1:15:00::","call_ids":["semeval2013.d000.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"china%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.004845,"input_uncached_usd":0.0028950000000000004,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.971437374944799}
{"item_id":"semeval2013.d000.s013.t003","gold_sense_keys":["bush_administration%1:14:01::"],"candidates":[{"index":1,"sense_key":"bush_administration%1:14:00::","synset_id":"bush_administration.n.01"},{"index":2,"sense_key":"bush_administration%1:14:01::","synset_id":"bush_administration.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bush_administration%1:14:01::","call_ids":["semeval2013.d000.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bush_administration%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.0052899999999999996,"input_uncached_usd":0.00232,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.438196958042681}
{"item_id":"semeval2013.d000.s013.t004","gold_sense_keys":["official%1:18:01::"],"candidates":[{"index":1,"sense_key":"official%1:18:01::","synset_id":"official.n.01"},{"index":2,"sense_key":"official%1:18:00::","synset_id":"official.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"official%1:18:01::","call_ids":["semeval2013.d000.s013.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"official%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.0040149999999999995,"input_uncached_usd":0.002395,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9090672089951113}
{"item_id":"semeval2013.d000.s013.t005","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":3,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d000.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.004445,"input_uncached_usd":0.0030050000000000003,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4719104579417035}
{"item_id":"semeval2013.d000.s014.t000","gold_sense_keys":["united_states%1:14:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"united_states%1:14:00::","call_ids":["semeval2013.d000.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"united_states%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":382,"reasoning_output_tokens":366},"cost":{"currency":"USD","total_usd":0.014105,"input_uncached_usd":0.002645,"input_cached_usd":0.0,"output_usd":0.01146,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.6692643749993294}
{"item_id":"semeval2013.d000.s014.t001","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":4,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":5,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":6,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":7,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":8,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":9,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d000.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":800,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":93},"cost":{"currency":"USD","total_usd":0.00727,"input_uncached_usd":0.004,"input_cached_usd":0.0,"output_usd":0.00327,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4208800840424374}
{"item_id":"semeval2013.d000.s014.t004","gold_sense_keys":["text%1:10:00::","text%1:10:01::"],"candidates":[{"index":1,"sense_key":"text%1:10:00::","synset_id":"text.n.01"},{"index":2,"sense_key":"text%1:10:03::","synset_id":"text.n.02"},{"index":3,"sense_key":"text%1:10:02::","synset_id":"textbook.n.01"},{"index":4,"sense_key":"text%1:10:01::","synset_id":"text.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"text%1:10:00::","call_ids":["semeval2013.d000.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"text%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":603,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":97},"cost":{"currency":"USD","total_usd":0.006405000000000001,"input_uncached_usd":0.0030150000000000003,"input_cached_usd":0.0,"output_usd":0.0033900000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.083322875085287}
{"item_id":"semeval2013.d000.s014.t005","gold_sense_keys":["cut%1:04:05::"],"candidates":[{"index":1,"sense_key":"cut%1:26:01::","synset_id":"cut.n.04"},{"index":2,"sense_key":"cut%1:04:05::","synset_id":"cut.n.19"},{"index":3,"sense_key":"cut%1:10:04::","synset_id":"cut.n.08"},{"index":4,"sense_key":"cut%1:04:09::","synset_id":"cut.n.15"},{"index":5,"sense_key":"cut%1:10:02::","synset_id":"stinger.n.02"},{"index":6,"sense_key":"cut%1:10:01::","synset_id":"deletion.n.03"},{"index":7,"sense_key":"cut%1:04:07::","synset_id":"baseball_swing.n.01"},{"index":8,"sense_key":"cut%1:04:01::","synset_id":"cut.n.17"},{"index":9,"sense_key":"cut%1:04:00::","synset_id":"cut.n.18"},{"index":10,"sense_key":"cut%1:04:02::","synset_id":"cut.n.16"},{"index":11,"sense_key":"cut%1:04:08::","synset_id":"cut.n.14"},{"index":12,"sense_key":"cut%1:26:00::","synset_id":"cut.n.05"},{"index":13,"sense_key":"cut%1:21:00::","synset_id":"cut.n.01"},{"index":14,"sense_key":"cut%1:10:00::","synset_id":"cut.n.02"},{"index":15,"sense_key":"cut%1:04:06::","synset_id":"cut.n.20"},{"index":16,"sense_key":"cut%1:13:00::","synset_id":"cut.n.06"},{"index":17,"sense_key":"cut%1:09:00::","synset_id":"cut.n.10"},{"index":18,"sense_key":"cut%1:06:00::","synset_id":"cut.n.03"},{"index":19,"sense_key":"cut%1:04:03::","synset_id":"snub.n.02"},{"index":20,"sense_key":"cut%1:06:01::","synset_id":"cut.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cut%1:04:05::","call_ids":["semeval2013.d000.s014.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cut%1:04:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1201,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.008945000000000002,"input_uncached_usd":0.006005000000000001,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.195912374998443}
{"item_id":"semeval2013.d000.s014.t006","gold_sense_keys":["observer%1:18:01::"],"candidates":[{"index":1,"sense_key":"observer%1:18:00::","synset_id":"perceiver.n.01"},{"index":2,"sense_key":"observer%1:18:01::","synset_id":"observer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"observer%1:18:01::","call_ids":["semeval2013.d000.s014.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"observer%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018225,"input_uncached_usd":0.0023850000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.82653041696176}
{"item_id":"semeval2013.d000.s015.t000","gold_sense_keys":["comment%1:10:00::"],"candidates":[{"index":1,"sense_key":"comment%1:10:02::","synset_id":"gossip.n.02"},{"index":2,"sense_key":"comment%1:10:01::","synset_id":"comment.n.02"},{"index":3,"sense_key":"comment%1:10:00::","synset_id":"remark.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"comment%1:10:00::","call_ids":["semeval2013.d000.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"comment%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":561,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.0042450000000000005,"input_uncached_usd":0.002805,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1540017079096287}
{"item_id":"semeval2013.d000.s015.t001","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"},{"index":2,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"},{"index":3,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"},{"index":4,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"hour%1:28:00::","call_ids":["semeval2013.d000.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":575,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.004525,"input_uncached_usd":0.0028750000000000004,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.600129875005223}
{"item_id":"semeval2013.d000.s015.t003","gold_sense_keys":["america%1:15:00::"],"candidates":[{"index":1,"sense_key":"america%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"america%1:17:00::","synset_id":"america.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"america%1:15:00::","call_ids":["semeval2013.d000.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"america%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.00456,"input_uncached_usd":0.0025800000000000003,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6353558750124648}
{"item_id":"semeval2013.d000.s015.t004","gold_sense_keys":["climate%1:26:00::"],"candidates":[{"index":1,"sense_key":"climate%1:26:00::","synset_id":"climate.n.01"},{"index":2,"sense_key":"climate%1:26:01::","synset_id":"climate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"climate%1:26:00::","call_ids":["semeval2013.d000.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"climate%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.00604,"input_uncached_usd":0.0025900000000000003,"input_cached_usd":0.0,"output_usd":0.00345,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7547015419695526}
{"item_id":"semeval2013.d000.s015.t007","gold_sense_keys":["week%1:28:02::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"week%1:28:02::","call_ids":["semeval2013.d000.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"week%1:28:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":234,"reasoning_output_tokens":218},"cost":{"currency":"USD","total_usd":0.009655,"input_uncached_usd":0.002635,"input_cached_usd":0.0,"output_usd":0.00702,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.72302299994044}
{"item_id":"semeval2013.d000.s015.t008","gold_sense_keys":["united_states%1:14:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"united_states%1:14:00::","call_ids":["semeval2013.d000.s015.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"united_states%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":541,"cached_input_tokens":0,"output_tokens":175,"reasoning_output_tokens":159},"cost":{"currency":"USD","total_usd":0.007955,"input_uncached_usd":0.0027050000000000004,"input_cached_usd":0.0,"output_usd":0.00525,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8706900000106543}
{"item_id":"semeval2013.d000.s015.t009","gold_sense_keys":["china%1:15:00::"],"candidates":[{"index":1,"sense_key":"china%1:15:00::","synset_id":"china.n.01"},{"index":2,"sense_key":"china%1:06:01::","synset_id":"chinaware.n.01"},{"index":3,"sense_key":"china%1:15:01::","synset_id":"taiwan.n.01"},{"index":4,"sense_key":"china%1:06:00::","synset_id":"china.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"china%1:15:00::","call_ids":["semeval2013.d000.s015.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"china%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00473,"input_uncached_usd":0.0029600000000000004,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7046687500551343}
{"item_id":"semeval2013.d000.s016.t001","gold_sense_keys":["world%1:17:00::"],"candidates":[{"index":1,"sense_key":"world%1:17:00::","synset_id":"earth.n.01"},{"index":2,"sense_key":"world%1:17:02::","synset_id":"world.n.06"},{"index":3,"sense_key":"world%1:17:01::","synset_id":"universe.n.01"},{"index":4,"sense_key":"world%1:09:00::","synset_id":"world.n.03"},{"index":5,"sense_key":"world%1:14:01::","synset_id":"world.n.02"},{"index":6,"sense_key":"world%1:05:00::","synset_id":"world.n.08"},{"index":7,"sense_key":"world%1:09:01::","synset_id":"worldly_concern.n.01"},{"index":8,"sense_key":"world%1:14:02::","synset_id":"populace.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"world%1:14:01::","call_ids":["semeval2013.d000.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"world%1:14:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":812,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0199,"input_uncached_usd":0.00406,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":543.9417275419692}
{"item_id":"semeval2013.d000.s017.t000","gold_sense_keys":["battle%1:04:02::"],"candidates":[{"index":1,"sense_key":"battle%1:04:02::","synset_id":"conflict.n.01"},{"index":2,"sense_key":"battle%1:04:00::","synset_id":"battle.n.01"},{"index":3,"sense_key":"battle%1:04:01::","synset_id":"struggle.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"battle%1:04:02::","call_ids":["semeval2013.d000.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"battle%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":627,"cached_input_tokens":0,"output_tokens":390,"reasoning_output_tokens":374},"cost":{"currency":"USD","total_usd":0.014835000000000001,"input_uncached_usd":0.003135,"input_cached_usd":0.0,"output_usd":0.0117,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.25900270906277}
{"item_id":"semeval2013.d000.s017.t001","gold_sense_keys":["economy%1:14:00::"],"candidates":[{"index":1,"sense_key":"economy%1:07:00::","synset_id":"economy.n.03"},{"index":2,"sense_key":"economy%1:04:00::","synset_id":"economy.n.04"},{"index":3,"sense_key":"economy%1:09:01::","synset_id":"economy.n.02"},{"index":4,"sense_key":"economy%1:14:00::","synset_id":"economy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"economy%1:14:00::","call_ids":["semeval2013.d000.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"economy%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.004595,"input_uncached_usd":0.0030350000000000004,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.702574040973559}
{"item_id":"semeval2013.d000.s017.t003","gold_sense_keys":["china%1:15:00::"],"candidates":[{"index":1,"sense_key":"china%1:15:00::","synset_id":"china.n.01"},{"index":2,"sense_key":"china%1:06:01::","synset_id":"chinaware.n.01"},{"index":3,"sense_key":"china%1:06:00::","synset_id":"china.n.02"},{"index":4,"sense_key":"china%1:15:01::","synset_id":"taiwan.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"china%1:15:00::","call_ids":["semeval2013.d000.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"china%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.004905,"input_uncached_usd":0.003105,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6982962910551578}
{"item_id":"semeval2013.d000.s017.t004","gold_sense_keys":["united_states%1:14:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"united_states%1:15:00::","call_ids":["semeval2013.d000.s017.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"united_states%1:15:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":348,"reasoning_output_tokens":332},"cost":{"currency":"USD","total_usd":0.01329,"input_uncached_usd":0.00285,"input_cached_usd":0.0,"output_usd":0.01044,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.080097959027626}
{"item_id":"semeval2013.d000.s017.t005","gold_sense_keys":["obligation%1:04:00::"],"candidates":[{"index":1,"sense_key":"obligation%1:21:00::","synset_id":"debt_instrument.n.01"},{"index":2,"sense_key":"obligation%1:10:00::","synset_id":"obligation.n.05"},{"index":3,"sense_key":"obligation%1:26:00::","synset_id":"obligation.n.02"},{"index":4,"sense_key":"obligation%1:04:00::","synset_id":"duty.n.01"},{"index":5,"sense_key":"obligation%1:24:00::","synset_id":"obligation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"obligation%1:04:00::","call_ids":["semeval2013.d000.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"obligation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":673,"cached_input_tokens":0,"output_tokens":757,"reasoning_output_tokens":735},"cost":{"currency":"USD","total_usd":0.026075,"input_uncached_usd":0.0033650000000000004,"input_cached_usd":0.0,"output_usd":0.02271,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.33436779200565}
{"item_id":"semeval2013.d000.s017.t006","gold_sense_keys":["nation%1:14:00::"],"candidates":[{"index":1,"sense_key":"nation%1:14:02::","synset_id":"nation.n.04"},{"index":2,"sense_key":"nation%1:18:00::","synset_id":"nation.n.03"},{"index":3,"sense_key":"nation%1:14:00::","synset_id":"state.n.04"},{"index":4,"sense_key":"nation%1:14:01::","synset_id":"nation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"nation%1:14:00::","call_ids":["semeval2013.d000.s017.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"nation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.005385000000000001,"input_uncached_usd":0.0031950000000000004,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.335940832970664}
{"item_id":"semeval2013.d000.s017.t007","gold_sense_keys":["world%1:17:00::"],"candidates":[{"index":1,"sense_key":"world%1:17:00::","synset_id":"earth.n.01"},{"index":2,"sense_key":"world%1:17:02::","synset_id":"world.n.06"},{"index":3,"sense_key":"world%1:14:01::","synset_id":"world.n.02"},{"index":4,"sense_key":"world%1:17:01::","synset_id":"universe.n.01"},{"index":5,"sense_key":"world%1:14:02::","synset_id":"populace.n.01"},{"index":6,"sense_key":"world%1:09:01::","synset_id":"worldly_concern.n.01"},{"index":7,"sense_key":"world%1:05:00::","synset_id":"world.n.08"},{"index":8,"sense_key":"world%1:09:00::","synset_id":"world.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"world%1:17:00::","call_ids":["semeval2013.d000.s017.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"world%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":844,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02006,"input_uncached_usd":0.004220000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.769575666985475}
{"item_id":"semeval2013.d000.s017.t009","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":2,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":3,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":4,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"country%1:15:00::","synset_id":"country.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s017.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":696,"cached_input_tokens":0,"output_tokens":141,"reasoning_output_tokens":125},"cost":{"currency":"USD","total_usd":0.007710000000000001,"input_uncached_usd":0.0034800000000000005,"input_cached_usd":0.0,"output_usd":0.00423,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.36640716693364}
{"item_id":"semeval2013.d000.s017.t010","gold_sense_keys":["imperative%1:04:00::"],"candidates":[{"index":1,"sense_key":"imperative%1:04:00::","synset_id":"imperative.n.02"},{"index":2,"sense_key":"imperative%1:24:00::","synset_id":"imperative_mood.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"imperative%1:04:00::","call_ids":["semeval2013.d000.s017.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"imperative%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.00429,"input_uncached_usd":0.0025800000000000003,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4458820420550182}
{"item_id":"semeval2013.d000.s018.t002","gold_sense_keys":["cost%1:21:00::"],"candidates":[{"index":1,"sense_key":"cost%1:21:00::","synset_id":"cost.n.01"},{"index":2,"sense_key":"cost%1:07:01::","synset_id":"price.n.03"},{"index":3,"sense_key":"cost%1:07:00::","synset_id":"monetary_value.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cost%1:21:00::","call_ids":["semeval2013.d000.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cost%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01882,"input_uncached_usd":0.0029800000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.91890337492805}
{"item_id":"semeval2013.d000.s018.t004","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":2,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":3,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:00::","synset_id":"country.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.006945,"input_uncached_usd":0.003465,"input_cached_usd":0.0,"output_usd":0.00348,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.631051291944459}
{"item_id":"semeval2013.d000.s018.t005","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":2,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":3,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":4,"sense_key":"action%1:04:00::","synset_id":"action.n.10"},{"index":5,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":6,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":7,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":8,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":9,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":10,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d000.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":900,"cached_input_tokens":0,"output_tokens":155,"reasoning_output_tokens":139},"cost":{"currency":"USD","total_usd":0.009150000000000002,"input_uncached_usd":0.0045000000000000005,"input_cached_usd":0.0,"output_usd":0.0046500000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5723933749832213}
{"item_id":"semeval2013.d000.s019.t003","gold_sense_keys":["central_america%1:14:00::"],"candidates":[{"index":1,"sense_key":"central_america%1:15:00::","synset_id":"central_america.n.01"},{"index":2,"sense_key":"central_america%1:14:00::","synset_id":"central_america.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"central_america%1:15:00::","call_ids":["semeval2013.d000.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"central_america%1:15:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":431,"reasoning_output_tokens":416},"cost":{"currency":"USD","total_usd":0.015595000000000001,"input_uncached_usd":0.0026650000000000003,"input_cached_usd":0.0,"output_usd":0.01293,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.404728916008025}
{"item_id":"semeval2013.d000.s020.t000","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":3,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":4,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":5,"sense_key":"country%1:15:00::","synset_id":"country.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":122},"cost":{"currency":"USD","total_usd":0.007595000000000001,"input_uncached_usd":0.003455,"input_cached_usd":0.0,"output_usd":0.0041400000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.828787292004563}
{"item_id":"semeval2013.d000.s020.t001","gold_sense_keys":["impact%1:19:00::"],"candidates":[{"index":1,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":2,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":3,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"},{"index":4,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"impact%1:19:00::","call_ids":["semeval2013.d000.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"impact%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00472,"input_uncached_usd":0.00301,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3672203751048073}
{"item_id":"semeval2013.d000.s020.t005","gold_sense_keys":["participant%1:18:00::"],"candidates":[{"index":1,"sense_key":"participant%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"participant%1:18:00::","synset_id":"participant.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"participant%1:18:00::","call_ids":["semeval2013.d000.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"participant%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.004945,"input_uncached_usd":0.0024850000000000002,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.258985292050056}
{"item_id":"semeval2013.d000.s020.t007","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":4,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":5,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":6,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":7,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":8,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d000.s020.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":836,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.00604,"input_uncached_usd":0.0041800000000000006,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7955122909042984}
{"item_id":"semeval2013.d000.s021.t000","gold_sense_keys":["member%1:14:00::"],"candidates":[{"index":1,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":2,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"},{"index":5,"sense_key":"member%1:24:00::","synset_id":"member.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"member%1:14:00::","call_ids":["semeval2013.d000.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"member%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":85},"cost":{"currency":"USD","total_usd":0.0064600000000000005,"input_uncached_usd":0.0034600000000000004,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.482361000031233}
{"item_id":"semeval2013.d000.s021.t001","gold_sense_keys":["draft%1:10:00::"],"candidates":[{"index":1,"sense_key":"draft%1:21:00::","synset_id":"draft.n.01"},{"index":2,"sense_key":"draft%1:13:00::","synset_id":"draft.n.04"},{"index":3,"sense_key":"draft%1:04:03::","synset_id":"conscription.n.01"},{"index":4,"sense_key":"draft%1:04:00::","synset_id":"draft.n.11"},{"index":5,"sense_key":"draft%1:06:02::","synset_id":"draft.n.07"},{"index":6,"sense_key":"draft%1:19:00::","synset_id":"draft.n.02"},{"index":7,"sense_key":"draft%1:07:00::","synset_id":"draft.n.06"},{"index":8,"sense_key":"draft%1:06:01::","synset_id":"draft.n.03"},{"index":9,"sense_key":"draft%1:04:02::","synset_id":"gulp.n.01"},{"index":10,"sense_key":"draft%1:10:00::","synset_id":"draft.n.05"},{"index":11,"sense_key":"draft%1:06:00::","synset_id":"draft.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"draft%1:10:00::","call_ids":["semeval2013.d000.s021.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"draft%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":847,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.006095,"input_uncached_usd":0.004235,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9357957079773769}
{"item_id":"semeval2013.d000.s021.t003","gold_sense_keys":["proposal%1:10:00::"],"candidates":[{"index":1,"sense_key":"proposal%1:10:01::","synset_id":"proposal.n.03"},{"index":2,"sense_key":"proposal%1:10:02::","synset_id":"marriage_proposal.n.01"},{"index":3,"sense_key":"proposal%1:10:00::","synset_id":"proposal.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"proposal%1:10:00::","call_ids":["semeval2013.d000.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"proposal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.00425,"input_uncached_usd":0.00269,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7185832089744508}
{"item_id":"semeval2013.d000.s021.t005","gold_sense_keys":["morning%1:28:00::"],"candidates":[{"index":1,"sense_key":"morning%1:28:01::","synset_id":"dawn.n.01"},{"index":2,"sense_key":"morning%1:11:00::","synset_id":"dawn.n.02"},{"index":3,"sense_key":"morning%1:10:00::","synset_id":"good_morning.n.01"},{"index":4,"sense_key":"morning%1:28:00::","synset_id":"morning.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"morning%1:28:00::","call_ids":["semeval2013.d000.s021.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"morning%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":216,"reasoning_output_tokens":200},"cost":{"currency":"USD","total_usd":0.00953,"input_uncached_usd":0.00305,"input_cached_usd":0.0,"output_usd":0.0064800000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6442392090102658}
{"item_id":"semeval2013.d000.s022.t000","gold_sense_keys":["climate%1:26:00::"],"candidates":[{"index":1,"sense_key":"climate%1:26:01::","synset_id":"climate.n.02"},{"index":2,"sense_key":"climate%1:26:00::","synset_id":"climate.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"climate%1:26:00::","call_ids":["semeval2013.d000.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"climate%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.0054150000000000005,"input_uncached_usd":0.0027150000000000004,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.564506790949963}
{"item_id":"semeval2013.d000.s022.t001","gold_sense_keys":["negotiation%1:10:00::"],"candidates":[{"index":1,"sense_key":"negotiation%1:10:00::","synset_id":"negotiation.n.01"},{"index":2,"sense_key":"negotiation%1:04:00::","synset_id":"negotiation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"negotiation%1:10:00::","call_ids":["semeval2013.d000.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"negotiation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018475,"input_uncached_usd":0.002635,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.970454042078927}
{"item_id":"semeval2013.d000.s022.t002","gold_sense_keys":["behalf%1:04:00::"],"candidates":[{"index":1,"sense_key":"behalf%1:04:00::","synset_id":"behalf.n.01"},{"index":2,"sense_key":"behalf%1:07:00::","synset_id":"behalf.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"behalf%1:04:00::","call_ids":["semeval2013.d000.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"behalf%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":541,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0044150000000000005,"input_uncached_usd":0.0027050000000000004,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.636805749963969}
{"item_id":"semeval2013.d000.s022.t004","gold_sense_keys":["pressure%1:07:00::"],"candidates":[{"index":1,"sense_key":"pressure%1:26:00::","synset_id":"imperativeness.n.01"},{"index":2,"sense_key":"pressure%1:19:01::","synset_id":"atmospheric_pressure.n.01"},{"index":3,"sense_key":"pressure%1:09:00::","synset_id":"pressure.n.05"},{"index":4,"sense_key":"pressure%1:04:00::","synset_id":"press.n.09"},{"index":5,"sense_key":"pressure%1:26:01::","synset_id":"pressure.n.06"},{"index":6,"sense_key":"pressure%1:19:00::","synset_id":"pressure.n.01"},{"index":7,"sense_key":"pressure%1:07:00::","synset_id":"pressure.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"pressure%1:07:00::","call_ids":["semeval2013.d000.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"pressure%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":97},"cost":{"currency":"USD","total_usd":0.00715,"input_uncached_usd":0.0037900000000000004,"input_cached_usd":0.0,"output_usd":0.00336,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7459679999155924}
{"item_id":"semeval2013.d000.s022.t005","gold_sense_keys":["process%1:04:00::"],"candidates":[{"index":1,"sense_key":"process%1:08:00::","synset_id":"process.n.05"},{"index":2,"sense_key":"process%1:09:01::","synset_id":"process.n.04"},{"index":3,"sense_key":"process%1:04:00::","synset_id":"procedure.n.01"},{"index":4,"sense_key":"process%1:03:00::","synset_id":"process.n.06"},{"index":5,"sense_key":"process%1:09:00::","synset_id":"process.n.02"},{"index":6,"sense_key":"process%1:10:00::","synset_id":"summons.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"process%1:04:00::","call_ids":["semeval2013.d000.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"process%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":745,"cached_input_tokens":0,"output_tokens":181,"reasoning_output_tokens":166},"cost":{"currency":"USD","total_usd":0.009155,"input_uncached_usd":0.0037250000000000004,"input_cached_usd":0.0,"output_usd":0.00543,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6073365410557017}
{"item_id":"semeval2013.d000.s022.t006","gold_sense_keys":["action%1:04:04::"],"candidates":[{"index":1,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":4,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":5,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":6,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":7,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":8,"sense_key":"action%1:04:00::","synset_id":"action.n.10"},{"index":9,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":10,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"action%1:04:04::","call_ids":["semeval2013.d000.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"action%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":900,"cached_input_tokens":0,"output_tokens":211,"reasoning_output_tokens":195},"cost":{"currency":"USD","total_usd":0.010830000000000001,"input_uncached_usd":0.0045000000000000005,"input_cached_usd":0.0,"output_usd":0.0063300000000000006,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9133653750177473}
{"item_id":"semeval2013.d000.s022.t007","gold_sense_keys":["china%1:15:00::"],"candidates":[{"index":1,"sense_key":"china%1:15:00::","synset_id":"china.n.01"},{"index":2,"sense_key":"china%1:06:00::","synset_id":"china.n.02"},{"index":3,"sense_key":"china%1:15:01::","synset_id":"taiwan.n.01"},{"index":4,"sense_key":"china%1:06:01::","synset_id":"chinaware.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"china%1:15:00::","call_ids":["semeval2013.d000.s022.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"china%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.004705,"input_uncached_usd":0.003085,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8938806670485064}
{"item_id":"semeval2013.d000.s023.t001","gold_sense_keys":["urgency%1:07:00::","urgency%1:26:00::"],"candidates":[{"index":1,"sense_key":"urgency%1:26:01::","synset_id":"urgency.n.03"},{"index":2,"sense_key":"urgency%1:07:00::","synset_id":"urgency.n.02"},{"index":3,"sense_key":"urgency%1:26:00::","synset_id":"urgency.n.01"},{"index":4,"sense_key":"urgency%1:10:00::","synset_id":"importunity.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"urgency%1:07:00::","call_ids":["semeval2013.d000.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"urgency%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018865,"input_uncached_usd":0.0030250000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.48285216698423}
{"item_id":"semeval2013.d000.s023.t003","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":2,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"},{"index":3,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":4,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"fact%1:09:01::","call_ids":["semeval2013.d000.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"fact%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018945,"input_uncached_usd":0.003105,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.962192542036064}
{"item_id":"semeval2013.d000.s023.t004","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":2,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":3,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":4,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":5,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":6,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":7,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":8,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"},{"index":11,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"issue%1:09:01::","call_ids":["semeval2013.d000.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":990,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.006960000000000001,"input_uncached_usd":0.00495,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5078367500100285}
{"item_id":"semeval2013.d000.s023.t005","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"},{"index":4,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":5,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d000.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":840,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.0064800000000000005,"input_uncached_usd":0.004200000000000001,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9372642079833895}
{"item_id":"semeval2013.d000.s024.t000","gold_sense_keys":["official%1:18:01::"],"candidates":[{"index":1,"sense_key":"official%1:18:00::","synset_id":"official.n.02"},{"index":2,"sense_key":"official%1:18:01::","synset_id":"official.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"official%1:18:01::","call_ids":["semeval2013.d000.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"official%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.004535000000000001,"input_uncached_usd":0.002645,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.311117041972466}
{"item_id":"semeval2013.d000.s024.t001","gold_sense_keys":["minister%1:18:02::"],"candidates":[{"index":1,"sense_key":"minister%1:18:02::","synset_id":"minister.n.02"},{"index":2,"sense_key":"minister%1:04:00::","synset_id":"minister.n.04"},{"index":3,"sense_key":"minister%1:18:00::","synset_id":"curate.n.01"},{"index":4,"sense_key":"minister%1:18:01::","synset_id":"minister.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"minister%1:18:02::","call_ids":["semeval2013.d000.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"minister%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.00524,"input_uncached_usd":0.00302,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9378614170709625}
{"item_id":"semeval2013.d000.s024.t002","gold_sense_keys":["plane%1:06:01::"],"candidates":[{"index":1,"sense_key":"plane%1:25:00::","synset_id":"plane.n.02"},{"index":2,"sense_key":"plane%1:26:00::","synset_id":"plane.n.03"},{"index":3,"sense_key":"plane%1:06:01::","synset_id":"airplane.n.01"},{"index":4,"sense_key":"plane%1:06:00::","synset_id":"plane.n.05"},{"index":5,"sense_key":"plane%1:06:02::","synset_id":"plane.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"plane%1:06:01::","call_ids":["semeval2013.d000.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"plane%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":689,"cached_input_tokens":0,"output_tokens":42,"reasoning_output_tokens":27},"cost":{"currency":"USD","total_usd":0.004705,"input_uncached_usd":0.003445,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2450416249921545}
{"item_id":"semeval2013.d000.s024.t003","gold_sense_keys":["hall%1:06:02::","hall%1:06:03::","hall%1:06:05::"],"candidates":[{"index":1,"sense_key":"hall%1:06:05::","synset_id":"anteroom.n.01"},{"index":2,"sense_key":"hall%1:06:08::","synset_id":"hall.n.12"},{"index":3,"sense_key":"hall%1:06:07::","synset_id":"mansion.n.02"},{"index":4,"sense_key":"hall%1:06:04::","synset_id":"dormitory.n.01"},{"index":5,"sense_key":"hall%1:06:06::","synset_id":"manor_hall.n.01"},{"index":6,"sense_key":"hall%1:06:03::","synset_id":"hallway.n.01"},{"index":7,"sense_key":"hall%1:18:01::","synset_id":"hall.n.09"},{"index":8,"sense_key":"hall%1:18:04::","synset_id":"hall.n.06"},{"index":9,"sense_key":"hall%1:06:01::","synset_id":"hall.n.13"},{"index":10,"sense_key":"hall%1:18:00::","synset_id":"hall.n.10"},{"index":11,"sense_key":"hall%1:06:02::","synset_id":"hall.n.03"},{"index":12,"sense_key":"hall%1:18:02::","synset_id":"hall.n.08"},{"index":13,"sense_key":"hall%1:18:03::","synset_id":"hall.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"hall%1:06:03::","call_ids":["semeval2013.d000.s024.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"hall%1:06:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":973,"cached_input_tokens":0,"output_tokens":167,"reasoning_output_tokens":151},"cost":{"currency":"USD","total_usd":0.009875000000000002,"input_uncached_usd":0.004865,"input_cached_usd":0.0,"output_usd":0.00501,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.324393707909621}
{"item_id":"semeval2013.d000.s024.t004","gold_sense_keys":["meeting%1:14:00::"],"candidates":[{"index":1,"sense_key":"meeting%1:14:00::","synset_id":"meeting.n.01"},{"index":2,"sense_key":"meeting%1:04:02::","synset_id":"merging.n.01"},{"index":3,"sense_key":"meeting%1:04:00::","synset_id":"meeting.n.04"},{"index":4,"sense_key":"meeting%1:14:01::","synset_id":"meeting.n.02"},{"index":5,"sense_key":"meeting%1:15:00::","synset_id":"confluence.n.01"},{"index":6,"sense_key":"meeting%1:11:00::","synset_id":"meeting.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"meeting%1:14:00::","call_ids":["semeval2013.d000.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"meeting%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":732,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.0066300000000000005,"input_uncached_usd":0.0036600000000000005,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1251838749740273}
{"item_id":"semeval2013.d000.s024.t006","gold_sense_keys":["claim%1:04:00::","claim%1:10:00::"],"candidates":[{"index":1,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":2,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":3,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":4,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":5,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"claim%1:04:00::","call_ids":["semeval2013.d000.s024.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"claim%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.034839999999999996,"input_uncached_usd":0.0034600000000000004,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":20.0713700420456}
{"item_id":"semeval2013.d000.s024.t007","gold_sense_keys":["week%1:28:00::"],"candidates":[{"index":1,"sense_key":"week%1:28:02::","synset_id":"week.n.03"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:00::","synset_id":"week.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"week%1:28:00::","call_ids":["semeval2013.d000.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"week%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.006265,"input_uncached_usd":0.0028150000000000002,"input_cached_usd":0.0,"output_usd":0.00345,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4707157909870148}
{"item_id":"semeval2013.d000.s025.t000","gold_sense_keys":["gathering%1:14:00::"],"candidates":[{"index":1,"sense_key":"gathering%1:04:01::","synset_id":"gather.n.02"},{"index":2,"sense_key":"gathering%1:06:00::","synset_id":"gather.n.01"},{"index":3,"sense_key":"gathering%1:14:00::","synset_id":"gathering.n.01"},{"index":4,"sense_key":"gathering%1:04:00::","synset_id":"assembly.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"gathering%1:14:00::","call_ids":["semeval2013.d000.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"gathering%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":1480,"reasoning_output_tokens":1452},"cost":{"currency":"USD","total_usd":0.047345000000000005,"input_uncached_usd":0.002945,"input_cached_usd":0.0,"output_usd":0.0444,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":31.882861249963753}
{"item_id":"semeval2013.d000.s025.t001","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":2,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"people%1:14:00::","call_ids":["semeval2013.d000.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":627,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.005175000000000001,"input_uncached_usd":0.003135,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.285753499949351}
{"item_id":"semeval2013.d000.s025.t002","gold_sense_keys":["day%1:28:00::"],"candidates":[{"index":1,"sense_key":"day%1:28:05::","synset_id":"day.n.06"},{"index":2,"sense_key":"day%1:28:07::","synset_id":"day.n.07"},{"index":3,"sense_key":"day%1:28:03::","synset_id":"day.n.02"},{"index":4,"sense_key":"day%1:26:00::","synset_id":"day.n.09"},{"index":5,"sense_key":"day%1:28:06::","synset_id":"sidereal_day.n.01"},{"index":6,"sense_key":"day%1:28:04::","synset_id":"day.n.05"},{"index":7,"sense_key":"day%1:28:00::","synset_id":"day.n.01"},{"index":8,"sense_key":"day%1:28:01::","synset_id":"day.n.03"},{"index":9,"sense_key":"day%1:18:00::","synset_id":"day.n.10"},{"index":10,"sense_key":"day%1:28:02::","synset_id":"day.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"day%1:28:00::","call_ids":["semeval2013.d000.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"day%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":912,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.007170000000000001,"input_uncached_usd":0.004560000000000001,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1341550829820335}
{"item_id":"semeval2013.d000.s025.t004","gold_sense_keys":["show%1:10:00::"],"candidates":[{"index":1,"sense_key":"show%1:10:03::","synset_id":"display.n.01"},{"index":2,"sense_key":"show%1:04:01::","synset_id":"appearance.n.06"},{"index":3,"sense_key":"show%1:04:00::","synset_id":"show.n.01"},{"index":4,"sense_key":"show%1:10:00::","synset_id":"show.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"show%1:10:00::","call_ids":["semeval2013.d000.s025.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"show%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":101},"cost":{"currency":"USD","total_usd":0.0065650000000000005,"input_uncached_usd":0.0030550000000000004,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3087458750233054}
{"item_id":"semeval2013.d000.s025.t005","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":2,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":3,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":4,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":5,"sense_key":"country%1:14:00::","synset_id":"state.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s025.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":179,"reasoning_output_tokens":163},"cost":{"currency":"USD","total_usd":0.008860000000000002,"input_uncached_usd":0.0034900000000000005,"input_cached_usd":0.0,"output_usd":0.00537,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.1205717080738395}
{"item_id":"semeval2013.d000.s025.t006","gold_sense_keys":["draft%1:10:00::"],"candidates":[{"index":1,"sense_key":"draft%1:06:00::","synset_id":"draft.n.08"},{"index":2,"sense_key":"draft%1:19:00::","synset_id":"draft.n.02"},{"index":3,"sense_key":"draft%1:04:00::","synset_id":"draft.n.11"},{"index":4,"sense_key":"draft%1:13:00::","synset_id":"draft.n.04"},{"index":5,"sense_key":"draft%1:07:00::","synset_id":"draft.n.06"},{"index":6,"sense_key":"draft%1:04:02::","synset_id":"gulp.n.01"},{"index":7,"sense_key":"draft%1:10:00::","synset_id":"draft.n.05"},{"index":8,"sense_key":"draft%1:06:01::","synset_id":"draft.n.03"},{"index":9,"sense_key":"draft%1:21:00::","synset_id":"draft.n.01"},{"index":10,"sense_key":"draft%1:06:02::","synset_id":"draft.n.07"},{"index":11,"sense_key":"draft%1:04:03::","synset_id":"conscription.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"draft%1:10:00::","call_ids":["semeval2013.d000.s025.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"draft%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":854,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.00724,"input_uncached_usd":0.00427,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.074401125079021}
{"item_id":"semeval2013.d000.s025.t007","gold_sense_keys":["proposal%1:10:00::"],"candidates":[{"index":1,"sense_key":"proposal%1:10:00::","synset_id":"proposal.n.01"},{"index":2,"sense_key":"proposal%1:10:01::","synset_id":"proposal.n.03"},{"index":3,"sense_key":"proposal%1:10:02::","synset_id":"marriage_proposal.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"proposal%1:10:00::","call_ids":["semeval2013.d000.s025.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"proposal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.005220000000000001,"input_uncached_usd":0.0027300000000000002,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9130552089773118}
{"item_id":"semeval2013.d000.s025.t008","gold_sense_keys":["paper%1:10:00::"],"candidates":[{"index":1,"sense_key":"paper%1:10:02::","synset_id":"paper.n.05"},{"index":2,"sense_key":"paper%1:10:03::","synset_id":"newspaper.n.01"},{"index":3,"sense_key":"paper%1:10:00::","synset_id":"paper.n.04"},{"index":4,"sense_key":"paper%1:06:00::","synset_id":"newspaper.n.03"},{"index":5,"sense_key":"paper%1:10:01::","synset_id":"composition.n.08"},{"index":6,"sense_key":"paper%1:27:00::","synset_id":"paper.n.01"},{"index":7,"sense_key":"paper%1:14:00::","synset_id":"newspaper.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"paper%1:10:00::","call_ids":["semeval2013.d000.s025.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"paper%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":137},"cost":{"currency":"USD","total_usd":0.00818,"input_uncached_usd":0.0036200000000000004,"input_cached_usd":0.0,"output_usd":0.00456,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9126174999400973}
{"item_id":"semeval2013.d000.s025.t010","gold_sense_keys":["challenge%1:26:00::"],"candidates":[{"index":1,"sense_key":"challenge%1:10:02::","synset_id":"challenge.n.04"},{"index":2,"sense_key":"challenge%1:10:01::","synset_id":"challenge.n.03"},{"index":3,"sense_key":"challenge%1:26:00::","synset_id":"challenge.n.01"},{"index":4,"sense_key":"challenge%1:10:03::","synset_id":"challenge.n.05"},{"index":5,"sense_key":"challenge%1:10:00::","synset_id":"challenge.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"challenge%1:26:00::","call_ids":["semeval2013.d000.s025.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"challenge%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":603,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.004695,"input_uncached_usd":0.0030150000000000003,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6005445000482723}
{"item_id":"semeval2013.d000.s026.t000","gold_sense_keys":["intensity%1:07:00::"],"candidates":[{"index":1,"sense_key":"intensity%1:07:01::","synset_id":"saturation.n.04"},{"index":2,"sense_key":"intensity%1:07:00::","synset_id":"intensity.n.02"},{"index":3,"sense_key":"intensity%1:07:03::","synset_id":"intensity.n.01"},{"index":4,"sense_key":"intensity%1:07:02::","synset_id":"volume.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"intensity%1:07:00::","call_ids":["semeval2013.d000.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"intensity%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00481,"input_uncached_usd":0.00301,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.638659792020917}
{"item_id":"semeval2013.d000.s026.t001","gold_sense_keys":["minister%1:18:02::"],"candidates":[{"index":1,"sense_key":"minister%1:04:00::","synset_id":"minister.n.04"},{"index":2,"sense_key":"minister%1:18:02::","synset_id":"minister.n.02"},{"index":3,"sense_key":"minister%1:18:01::","synset_id":"minister.n.03"},{"index":4,"sense_key":"minister%1:18:00::","synset_id":"curate.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"minister%1:18:02::","call_ids":["semeval2013.d000.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"minister%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.0058850000000000005,"input_uncached_usd":0.0028850000000000004,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.690343708032742}
{"item_id":"semeval2013.d000.s027.t002","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"policy%1:09:00::","call_ids":["semeval2013.d000.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":118},"cost":{"currency":"USD","total_usd":0.00668,"input_uncached_usd":0.00269,"input_cached_usd":0.0,"output_usd":0.0039900000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6017233749153093}
{"item_id":"semeval2013.d000.s028.t000","gold_sense_keys":["heavyweight%1:18:00::"],"candidates":[{"index":1,"sense_key":"heavyweight%1:18:04::","synset_id":"heavyweight.n.01"},{"index":2,"sense_key":"heavyweight%1:18:02::","synset_id":"heavyweight.n.02"},{"index":3,"sense_key":"heavyweight%1:18:00::","synset_id":"colossus.n.02"},{"index":4,"sense_key":"heavyweight%1:18:03::","synset_id":"giant.n.04"},{"index":5,"sense_key":"heavyweight%1:18:01::","synset_id":"heavyweight.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"heavyweight%1:18:00::","call_ids":["semeval2013.d000.s028.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"heavyweight%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.004285000000000001,"input_uncached_usd":0.0025450000000000004,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6516309169819579}
{"item_id":"semeval2013.d001.s000.t001","gold_sense_keys":["israel%1:15:00::"],"candidates":[{"index":1,"sense_key":"israel%1:15:00::","synset_id":"israel.n.01"},{"index":2,"sense_key":"israel%1:15:01::","synset_id":"israel.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"israel%1:15:00::","call_ids":["semeval2013.d001.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"israel%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":327,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.003795,"input_uncached_usd":0.0016350000000000002,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5628907500067726}
{"item_id":"semeval2013.d001.s001.t002","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:01::","synset_id":"team.n.02"},{"index":2,"sense_key":"team%1:14:00::","synset_id":"team.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":335,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0035650000000000005,"input_uncached_usd":0.001675,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9663367089815438}
{"item_id":"semeval2013.d001.s001.t003","gold_sense_keys":["loss%1:04:00::"],"candidates":[{"index":1,"sense_key":"loss%1:21:00::","synset_id":"loss.n.06"},{"index":2,"sense_key":"loss%1:11:02::","synset_id":"passing.n.02"},{"index":3,"sense_key":"loss%1:04:00::","synset_id":"loss.n.03"},{"index":4,"sense_key":"loss%1:11:01::","synset_id":"loss.n.05"},{"index":5,"sense_key":"loss%1:07:00::","synset_id":"loss.n.04"},{"index":6,"sense_key":"loss%1:22:00::","synset_id":"loss.n.02"},{"index":7,"sense_key":"loss%1:11:00::","synset_id":"personnel_casualty.n.01"},{"index":8,"sense_key":"loss%1:21:01::","synset_id":"loss.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"loss%1:07:00::","call_ids":["semeval2013.d001.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"loss%1:07:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01887,"input_uncached_usd":0.00303,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.241847875062376}
{"item_id":"semeval2013.d001.s001.t004","gold_sense_keys":["performance%1:04:00::","performance%1:04:03::"],"candidates":[{"index":1,"sense_key":"performance%1:22:00::","synset_id":"operation.n.08"},{"index":2,"sense_key":"performance%1:04:00::","synset_id":"performance.n.03"},{"index":3,"sense_key":"performance%1:10:00::","synset_id":"performance.n.01"},{"index":4,"sense_key":"performance%1:04:01::","synset_id":"performance.n.02"},{"index":5,"sense_key":"performance%1:04:03::","synset_id":"performance.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"performance%1:04:03::","call_ids":["semeval2013.d001.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"performance%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.005765,"input_uncached_usd":0.002645,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5039152500685304}
{"item_id":"semeval2013.d001.s001.t005","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":334,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00335,"input_uncached_usd":0.00167,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.870525125064887}
{"item_id":"semeval2013.d001.s001.t006","gold_sense_keys":["point%1:23:01::"],"candidates":[{"index":1,"sense_key":"point%1:10:00::","synset_id":"period.n.07"},{"index":2,"sense_key":"point%1:25:00::","synset_id":"point.n.08"},{"index":3,"sense_key":"point%1:07:01::","synset_id":"point.n.14"},{"index":4,"sense_key":"point%1:26:00::","synset_id":"degree.n.02"},{"index":5,"sense_key":"point%1:06:02::","synset_id":"point.n.24"},{"index":6,"sense_key":"point%1:10:02::","synset_id":"point.n.20"},{"index":7,"sense_key":"point%1:10:04::","synset_id":"decimal_point.n.01"},{"index":8,"sense_key":"point%1:25:02::","synset_id":"point.n.09"},{"index":9,"sense_key":"point%1:10:01::","synset_id":"point.n.03"},{"index":10,"sense_key":"point%1:23:01::","synset_id":"point.n.10"},{"index":11,"sense_key":"point%1:17:02::","synset_id":"point.n.11"},{"index":12,"sense_key":"point%1:09:02::","synset_id":"point.n.07"},{"index":13,"sense_key":"point%1:06:03::","synset_id":"distributor_point.n.01"},{"index":14,"sense_key":"point%1:24:04::","synset_id":"point.n.13"},{"index":15,"sense_key":"point%1:21:00::","synset_id":"point.n.18"},{"index":16,"sense_key":"point%1:09:01::","synset_id":"detail.n.01"},{"index":17,"sense_key":"point%1:10:03::","synset_id":"item.n.01"},{"index":18,"sense_key":"point%1:06:00::","synset_id":"point.n.15"},{"index":19,"sense_key":"point%1:28:00::","synset_id":"point.n.06"},{"index":20,"sense_key":"point%1:15:00::","synset_id":"point.n.02"},{"index":21,"sense_key":"point%1:07:02::","synset_id":"point.n.23"},{"index":22,"sense_key":"point%1:09:00::","synset_id":"point.n.01"},{"index":23,"sense_key":"point%1:24:00::","synset_id":"compass_point.n.01"},{"index":24,"sense_key":"point%1:06:01::","synset_id":"point.n.25"},{"index":25,"sense_key":"point%1:07:00::","synset_id":"point.n.22"},{"index":26,"sense_key":"point%1:23:00::","synset_id":"point.n.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"point%1:23:01::","call_ids":["semeval2013.d001.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"point%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1339,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.008735,"input_uncached_usd":0.006695,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6570448750862852}
{"item_id":"semeval2013.d001.s001.t007","gold_sense_keys":["top%1:26:00::"],"candidates":[{"index":1,"sense_key":"top%1:06:01::","synset_id":"top.n.09"},{"index":2,"sense_key":"top%1:28:00::","synset_id":"top.n.04"},{"index":3,"sense_key":"top%1:26:00::","synset_id":"acme.n.01"},{"index":4,"sense_key":"top%1:06:02::","synset_id":"circus_tent.n.01"},{"index":5,"sense_key":"top%1:07:02::","synset_id":"top.n.06"},{"index":6,"sense_key":"top%1:06:03::","synset_id":"top.n.08"},{"index":7,"sense_key":"top%1:06:00::","synset_id":"top.n.10"},{"index":8,"sense_key":"top%1:15:00::","synset_id":"top.n.02"},{"index":9,"sense_key":"top%1:15:01::","synset_id":"top.n.01"},{"index":10,"sense_key":"top%1:06:04::","synset_id":"top.n.07"},{"index":11,"sense_key":"top%1:15:02::","synset_id":"peak.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"top%1:26:00::","call_ids":["semeval2013.d001.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"top%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":850,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02009,"input_uncached_usd":0.00425,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.176669541979209}
{"item_id":"semeval2013.d001.s001.t008","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":2,"sense_key":"group%1:09:00::","synset_id":"group.n.03"},{"index":3,"sense_key":"group%1:03:00::","synset_id":"group.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"group%1:03:00::","call_ids":["semeval2013.d001.s001.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":395,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.004045,"input_uncached_usd":0.001975,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7232283749617636}
{"item_id":"semeval2013.d001.s002.t000","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":360,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.00339,"input_uncached_usd":0.0018000000000000002,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4175055000232533}
{"item_id":"semeval2013.d001.s002.t001","gold_sense_keys":["absence%1:04:00::","absence%1:26:00::"],"candidates":[{"index":1,"sense_key":"absence%1:28:00::","synset_id":"absence.n.03"},{"index":2,"sense_key":"absence%1:26:01::","synset_id":"absence.n.04"},{"index":3,"sense_key":"absence%1:26:00::","synset_id":"absence.n.01"},{"index":4,"sense_key":"absence%1:04:00::","synset_id":"absence.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"absence%1:04:00::","call_ids":["semeval2013.d001.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"absence%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":407,"reasoning_output_tokens":391},"cost":{"currency":"USD","total_usd":0.01448,"input_uncached_usd":0.0022700000000000003,"input_cached_usd":0.0,"output_usd":0.01221,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.668650332954712}
{"item_id":"semeval2013.d001.s002.t002","gold_sense_keys":["involvement%1:04:00::"],"candidates":[{"index":1,"sense_key":"involvement%1:09:00::","synset_id":"interest.n.01"},{"index":2,"sense_key":"involvement%1:26:01::","synset_id":"participation.n.02"},{"index":3,"sense_key":"involvement%1:04:00::","synset_id":"engagement.n.07"},{"index":4,"sense_key":"involvement%1:26:00::","synset_id":"affair.n.02"},{"index":5,"sense_key":"involvement%1:24:00::","synset_id":"involvement.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"involvement%1:04:00::","call_ids":["semeval2013.d001.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"involvement%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":106},"cost":{"currency":"USD","total_usd":0.0062450000000000006,"input_uncached_usd":0.002585,"input_cached_usd":0.0,"output_usd":0.00366,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4698578750249}
{"item_id":"semeval2013.d001.s002.t004","gold_sense_keys":["end%1:09:00::","end%1:11:00::","end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:18:00::","synset_id":"end.n.08"},{"index":2,"sense_key":"end%1:11:00::","synset_id":"end.n.03"},{"index":3,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":4,"sense_key":"end%1:04:02::","synset_id":"end.n.14"},{"index":5,"sense_key":"end%1:15:02::","synset_id":"end.n.07"},{"index":6,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":7,"sense_key":"end%1:15:00::","synset_id":"end.n.01"},{"index":8,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":9,"sense_key":"end%1:15:01::","synset_id":"end.n.09"},{"index":10,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":11,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":12,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":13,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":14,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"end%1:11:00::","call_ids":["semeval2013.d001.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"end%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":898,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02033,"input_uncached_usd":0.00449,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.067840250092559}
{"item_id":"semeval2013.d001.s003.t001","gold_sense_keys":["sideline%1:15:00::"],"candidates":[{"index":1,"sense_key":"sideline%1:15:00::","synset_id":"sideline.n.01"},{"index":2,"sense_key":"sideline%1:04:00::","synset_id":"avocation.n.01"},{"index":3,"sense_key":"sideline%1:06:00::","synset_id":"sideline.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sideline%1:15:00::","call_ids":["semeval2013.d001.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sideline%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":170,"reasoning_output_tokens":155},"cost":{"currency":"USD","total_usd":0.007210000000000001,"input_uncached_usd":0.0021100000000000003,"input_cached_usd":0.0,"output_usd":0.0051,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.397827084059827}
{"item_id":"semeval2013.d001.s003.t002","gold_sense_keys":["lot%1:23:00::"],"candidates":[{"index":1,"sense_key":"lot%1:23:00::","synset_id":"batch.n.02"},{"index":2,"sense_key":"lot%1:18:00::","synset_id":"lot.n.07"},{"index":3,"sense_key":"lot%1:06:00::","synset_id":"draw.n.04"},{"index":4,"sense_key":"lot%1:14:01::","synset_id":"bunch.n.03"},{"index":5,"sense_key":"lot%1:14:00::","synset_id":"set.n.05"},{"index":6,"sense_key":"lot%1:15:00::","synset_id":"lot.n.02"},{"index":7,"sense_key":"lot%1:26:00::","synset_id":"fortune.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lot%1:23:00::","call_ids":["semeval2013.d001.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lot%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.004985,"input_uncached_usd":0.003305,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3262319579953328}
{"item_id":"semeval2013.d001.s003.t003","gold_sense_keys":["triumph%1:11:00::"],"candidates":[{"index":1,"sense_key":"triumph%1:11:00::","synset_id":"victory.n.01"},{"index":2,"sense_key":"triumph%1:12:00::","synset_id":"triumph.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"triumph%1:11:00::","call_ids":["semeval2013.d001.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"triumph%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.00401,"input_uncached_usd":0.002,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.406973582925275}
{"item_id":"semeval2013.d001.s004.t000","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":2,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":6,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":7,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":8,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":9,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":10,"sense_key":"game%1:04:04::","synset_id":"game.n.11"},{"index":11,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d001.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":779,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":103},"cost":{"currency":"USD","total_usd":0.007465,"input_uncached_usd":0.0038950000000000005,"input_cached_usd":0.0,"output_usd":0.0035700000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.446806167019531}
{"item_id":"semeval2013.d001.s004.t001","gold_sense_keys":["favor%1:07:00::"],"candidates":[{"index":1,"sense_key":"favor%1:12:00::","synset_id":"favor.n.04"},{"index":2,"sense_key":"favor%1:06:00::","synset_id":"party_favor.n.01"},{"index":3,"sense_key":"favor%1:04:00::","synset_id":"favor.n.01"},{"index":4,"sense_key":"favor%1:09:00::","synset_id":"favor.n.03"},{"index":5,"sense_key":"favor%1:07:00::","synset_id":"favor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"favor%1:07:00::","call_ids":["semeval2013.d001.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"favor%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.005215,"input_uncached_usd":0.002605,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0434994590468705}
{"item_id":"semeval2013.d001.s005.t000","gold_sense_keys":["basket%1:04:00::"],"candidates":[{"index":1,"sense_key":"basket%1:23:00::","synset_id":"basket.n.02"},{"index":2,"sense_key":"basket%1:06:01::","synset_id":"basket.n.03"},{"index":3,"sense_key":"basket%1:04:00::","synset_id":"basket.n.04"},{"index":4,"sense_key":"basket%1:06:00::","synset_id":"basket.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"basket%1:04:00::","call_ids":["semeval2013.d001.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"basket%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.00456,"input_uncached_usd":0.00255,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.642537916894071}
{"item_id":"semeval2013.d001.s005.t001","gold_sense_keys":["perimeter%1:25:00::"],"candidates":[{"index":1,"sense_key":"perimeter%1:07:00::","synset_id":"circumference.n.01"},{"index":2,"sense_key":"perimeter%1:25:01::","synset_id":"perimeter.n.02"},{"index":3,"sense_key":"perimeter%1:25:00::","synset_id":"margin.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"perimeter%1:25:00::","call_ids":["semeval2013.d001.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"perimeter%1:25:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":230,"reasoning_output_tokens":215},"cost":{"currency":"USD","total_usd":0.009255,"input_uncached_usd":0.0023550000000000003,"input_cached_usd":0.0,"output_usd":0.0069,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9307792090112343}
{"item_id":"semeval2013.d001.s005.t002","gold_sense_keys":["speculation%1:09:00::"],"candidates":[{"index":1,"sense_key":"speculation%1:09:01::","synset_id":"meditation.n.01"},{"index":2,"sense_key":"speculation%1:10:03::","synset_id":"guess.n.01"},{"index":3,"sense_key":"speculation%1:21:00::","synset_id":"speculation.n.03"},{"index":4,"sense_key":"speculation%1:09:00::","synset_id":"speculation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"speculation%1:09:00::","call_ids":["semeval2013.d001.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"speculation%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":2815,"reasoning_output_tokens":2769},"cost":{"currency":"USD","total_usd":0.08740500000000001,"input_uncached_usd":0.002955,"input_cached_usd":0.0,"output_usd":0.08445,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":46.492522999993525}
{"item_id":"semeval2013.d001.s006.t000","gold_sense_keys":["scorer%1:18:00::"],"candidates":[{"index":1,"sense_key":"scorer%1:18:02::","synset_id":"scorekeeper.n.01"},{"index":2,"sense_key":"scorer%1:18:01::","synset_id":"scorer.n.02"},{"index":3,"sense_key":"scorer%1:18:00::","synset_id":"scorer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"scorer%1:18:00::","call_ids":["semeval2013.d001.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"scorer%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.004275,"input_uncached_usd":0.002565,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.267706500017084}
{"item_id":"semeval2013.d001.s006.t001","gold_sense_keys":["end%1:09:00::","end%1:11:00::","end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":2,"sense_key":"end%1:15:02::","synset_id":"end.n.07"},{"index":3,"sense_key":"end%1:11:00::","synset_id":"end.n.03"},{"index":4,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":5,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":6,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":7,"sense_key":"end%1:15:01::","synset_id":"end.n.09"},{"index":8,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":9,"sense_key":"end%1:15:00::","synset_id":"end.n.01"},{"index":10,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":11,"sense_key":"end%1:04:02::","synset_id":"end.n.14"},{"index":12,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":13,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"},{"index":14,"sense_key":"end%1:18:00::","synset_id":"end.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"end%1:28:00::","call_ids":["semeval2013.d001.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"end%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1019,"cached_input_tokens":0,"output_tokens":206,"reasoning_output_tokens":190},"cost":{"currency":"USD","total_usd":0.011275,"input_uncached_usd":0.005095000000000001,"input_cached_usd":0.0,"output_usd":0.00618,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8576087919063866}
{"item_id":"semeval2013.d001.s007.t000","gold_sense_keys":["quarter%1:28:01::"],"candidates":[{"index":1,"sense_key":"quarter%1:15:00::","synset_id":"quarter.n.02"},{"index":2,"sense_key":"quarter%1:28:01::","synset_id":"quarter.n.03"},{"index":3,"sense_key":"quarter%1:23:01::","synset_id":"one-fourth.n.01"},{"index":4,"sense_key":"quarter%1:24:00::","synset_id":"quarter.n.07"},{"index":5,"sense_key":"quarter%1:23:04::","synset_id":"quarter.n.09"},{"index":6,"sense_key":"quarter%1:21:00::","synset_id":"quarter.n.10"},{"index":7,"sense_key":"quarter%1:18:00::","synset_id":"quarter.n.11"},{"index":8,"sense_key":"quarter%1:28:02::","synset_id":"quarter.n.06"},{"index":9,"sense_key":"quarter%1:06:01::","synset_id":"quarter.n.13"},{"index":10,"sense_key":"quarter%1:04:00::","synset_id":"quarter.n.14"},{"index":11,"sense_key":"quarter%1:28:03::","synset_id":"quarter.n.04"},{"index":12,"sense_key":"quarter%1:23:02::","synset_id":"quarter.n.08"},{"index":13,"sense_key":"quarter%1:06:00::","synset_id":"stern.n.01"},{"index":14,"sense_key":"quarter%1:28:00::","synset_id":"quarter.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"quarter%1:28:01::","call_ids":["semeval2013.d001.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"quarter%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":906,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.00669,"input_uncached_usd":0.00453,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7807147500570863}
{"item_id":"semeval2013.d001.s007.t001","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:01::","synset_id":"team.n.02"},{"index":2,"sense_key":"team%1:14:00::","synset_id":"team.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.003755,"input_uncached_usd":0.002285,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8134612910216674}
{"item_id":"semeval2013.d001.s007.t003","gold_sense_keys":["work%1:04:00::"],"candidates":[{"index":1,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":2,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":3,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":4,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"},{"index":5,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":6,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":7,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"work%1:04:00::","call_ids":["semeval2013.d001.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"work%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":115},"cost":{"currency":"USD","total_usd":0.00774,"input_uncached_usd":0.0038100000000000005,"input_cached_usd":0.0,"output_usd":0.00393,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.319550707936287}
{"item_id":"semeval2013.d001.s007.t004","gold_sense_keys":["whistle%1:10:00::"],"candidates":[{"index":1,"sense_key":"whistle%1:10:00::","synset_id":"whistle.n.02"},{"index":2,"sense_key":"whistle%1:06:00::","synset_id":"pennywhistle.n.01"},{"index":3,"sense_key":"whistle%1:06:02::","synset_id":"whistle.n.03"},{"index":4,"sense_key":"whistle%1:11:00::","synset_id":"whistle.n.01"},{"index":5,"sense_key":"whistle%1:06:01::","synset_id":"whistle.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"whistle%1:10:00::","call_ids":["semeval2013.d001.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"whistle%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.011170000000000001,"input_uncached_usd":0.00301,"input_cached_usd":0.0,"output_usd":0.00816,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.250722958007827}
{"item_id":"semeval2013.d001.s007.t006","gold_sense_keys":["return%1:04:00::","return%1:04:01::"],"candidates":[{"index":1,"sense_key":"return%1:11:02::","synset_id":"return.n.03"},{"index":2,"sense_key":"return%1:10:01::","synset_id":"tax_return.n.01"},{"index":3,"sense_key":"return%1:04:05::","synset_id":"return.n.12"},{"index":4,"sense_key":"return%1:21:00::","synset_id":"return.n.06"},{"index":5,"sense_key":"return%1:06:00::","synset_id":"return_key.n.01"},{"index":6,"sense_key":"return%1:04:01::","synset_id":"return.n.02"},{"index":7,"sense_key":"return%1:04:07::","synset_id":"reappearance.n.02"},{"index":8,"sense_key":"return%1:04:02::","synset_id":"restitution.n.03"},{"index":9,"sense_key":"return%1:04:00::","synset_id":"return.n.05"},{"index":10,"sense_key":"return%1:04:04::","synset_id":"return.n.11"},{"index":11,"sense_key":"return%1:04:06::","synset_id":"return.n.10"},{"index":12,"sense_key":"return%1:11:01::","synset_id":"recurrence.n.01"},{"index":13,"sense_key":"return%1:10:02::","synset_id":"rejoinder.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"return%1:04:01::","call_ids":["semeval2013.d001.s007.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"return%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":974,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":135},"cost":{"currency":"USD","total_usd":0.0094,"input_uncached_usd":0.00487,"input_cached_usd":0.0,"output_usd":0.00453,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.391213583992794}
{"item_id":"semeval2013.d001.s007.t007","gold_sense_keys":["home%1:15:02::"],"candidates":[{"index":1,"sense_key":"home%1:06:01::","synset_id":"home.n.09"},{"index":2,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":3,"sense_key":"home%1:14:00::","synset_id":"family.n.01"},{"index":4,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":5,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"},{"index":6,"sense_key":"home%1:15:03::","synset_id":"home.n.06"},{"index":7,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":8,"sense_key":"home%1:15:02::","synset_id":"home.n.03"},{"index":9,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"home%1:15:02::","call_ids":["semeval2013.d001.s007.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"home%1:15:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":813,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019875,"input_uncached_usd":0.004065,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.02849195804447}
{"item_id":"semeval2013.d001.s007.t008","gold_sense_keys":["lead%1:23:00::"],"candidates":[{"index":1,"sense_key":"lead%1:09:00::","synset_id":"lead.n.03"},{"index":2,"sense_key":"lead%1:06:03::","synset_id":"leash.n.01"},{"index":3,"sense_key":"lead%1:04:00::","synset_id":"lead.n.17"},{"index":4,"sense_key":"lead%1:07:01::","synset_id":"spark_advance.n.01"},{"index":5,"sense_key":"lead%1:25:00::","synset_id":"lead.n.05"},{"index":6,"sense_key":"lead%1:15:00::","synset_id":"lead.n.09"},{"index":7,"sense_key":"lead%1:18:00::","synset_id":"star.n.04"},{"index":8,"sense_key":"lead%1:06:00::","synset_id":"lead.n.15"},{"index":9,"sense_key":"lead%1:06:02::","synset_id":"lead.n.14"},{"index":10,"sense_key":"lead%1:06:01::","synset_id":"jumper_cable.n.01"},{"index":11,"sense_key":"lead%1:10:00::","synset_id":"lead.n.11"},{"index":12,"sense_key":"lead%1:10:02::","synset_id":"tip.n.03"},{"index":13,"sense_key":"lead%1:04:01::","synset_id":"lead.n.04"},{"index":14,"sense_key":"lead%1:23:00::","synset_id":"lead.n.07"},{"index":15,"sense_key":"lead%1:10:01::","synset_id":"lead.n.06"},{"index":16,"sense_key":"lead%1:07:02::","synset_id":"lead.n.01"},{"index":17,"sense_key":"lead%1:27:00::","synset_id":"lead.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"lead%1:23:00::","call_ids":["semeval2013.d001.s007.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"lead%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1110,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.00759,"input_uncached_usd":0.00555,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0007350830128416}
{"item_id":"semeval2013.d001.s008.t000","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:01::","synset_id":"team.n.02"},{"index":2,"sense_key":"team%1:14:00::","synset_id":"team.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.00379,"input_uncached_usd":0.00232,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5904471669346094}
{"item_id":"semeval2013.d001.s008.t001","gold_sense_keys":["point%1:23:01::"],"candidates":[{"index":1,"sense_key":"point%1:15:00::","synset_id":"point.n.02"},{"index":2,"sense_key":"point%1:10:00::","synset_id":"period.n.07"},{"index":3,"sense_key":"point%1:06:01::","synset_id":"point.n.25"},{"index":4,"sense_key":"point%1:25:02::","synset_id":"point.n.09"},{"index":5,"sense_key":"point%1:10:01::","synset_id":"point.n.03"},{"index":6,"sense_key":"point%1:28:00::","synset_id":"point.n.06"},{"index":7,"sense_key":"point%1:09:02::","synset_id":"point.n.07"},{"index":8,"sense_key":"point%1:07:00::","synset_id":"point.n.22"},{"index":9,"sense_key":"point%1:09:00::","synset_id":"point.n.01"},{"index":10,"sense_key":"point%1:10:02::","synset_id":"point.n.20"},{"index":11,"sense_key":"point%1:07:02::","synset_id":"point.n.23"},{"index":12,"sense_key":"point%1:07:01::","synset_id":"point.n.14"},{"index":13,"sense_key":"point%1:09:01::","synset_id":"detail.n.01"},{"index":14,"sense_key":"point%1:23:00::","synset_id":"point.n.17"},{"index":15,"sense_key":"point%1:06:02::","synset_id":"point.n.24"},{"index":16,"sense_key":"point%1:06:00::","synset_id":"point.n.15"},{"index":17,"sense_key":"point%1:23:01::","synset_id":"point.n.10"},{"index":18,"sense_key":"point%1:21:00::","synset_id":"point.n.18"},{"index":19,"sense_key":"point%1:24:00::","synset_id":"compass_point.n.01"},{"index":20,"sense_key":"point%1:25:00::","synset_id":"point.n.08"},{"index":21,"sense_key":"point%1:24:04::","synset_id":"point.n.13"},{"index":22,"sense_key":"point%1:06:03::","synset_id":"distributor_point.n.01"},{"index":23,"sense_key":"point%1:26:00::","synset_id":"degree.n.02"},{"index":24,"sense_key":"point%1:10:04::","synset_id":"decimal_point.n.01"},{"index":25,"sense_key":"point%1:17:02::","synset_id":"point.n.11"},{"index":26,"sense_key":"point%1:10:03::","synset_id":"item.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"point%1:23:01::","call_ids":["semeval2013.d001.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"point%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1468,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.0092,"input_uncached_usd":0.00734,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.680790749960579}
{"item_id":"semeval2013.d001.s008.t002","gold_sense_keys":["point%1:23:01::"],"candidates":[{"index":1,"sense_key":"point%1:25:02::","synset_id":"point.n.09"},{"index":2,"sense_key":"point%1:10:00::","synset_id":"period.n.07"},{"index":3,"sense_key":"point%1:07:01::","synset_id":"point.n.14"},{"index":4,"sense_key":"point%1:21:00::","synset_id":"point.n.18"},{"index":5,"sense_key":"point%1:23:00::","synset_id":"point.n.17"},{"index":6,"sense_key":"point%1:06:02::","synset_id":"point.n.24"},{"index":7,"sense_key":"point%1:07:00::","synset_id":"point.n.22"},{"index":8,"sense_key":"point%1:09:02::","synset_id":"point.n.07"},{"index":9,"sense_key":"point%1:26:00::","synset_id":"degree.n.02"},{"index":10,"sense_key":"point%1:06:00::","synset_id":"point.n.15"},{"index":11,"sense_key":"point%1:25:00::","synset_id":"point.n.08"},{"index":12,"sense_key":"point%1:10:04::","synset_id":"decimal_point.n.01"},{"index":13,"sense_key":"point%1:07:02::","synset_id":"point.n.23"},{"index":14,"sense_key":"point%1:23:01::","synset_id":"point.n.10"},{"index":15,"sense_key":"point%1:17:02::","synset_id":"point.n.11"},{"index":16,"sense_key":"point%1:24:04::","synset_id":"point.n.13"},{"index":17,"sense_key":"point%1:06:01::","synset_id":"point.n.25"},{"index":18,"sense_key":"point%1:15:00::","synset_id":"point.n.02"},{"index":19,"sense_key":"point%1:10:02::","synset_id":"point.n.20"},{"index":20,"sense_key":"point%1:09:01::","synset_id":"detail.n.01"},{"index":21,"sense_key":"point%1:28:00::","synset_id":"point.n.06"},{"index":22,"sense_key":"point%1:09:00::","synset_id":"point.n.01"},{"index":23,"sense_key":"point%1:10:01::","synset_id":"point.n.03"},{"index":24,"sense_key":"point%1:10:03::","synset_id":"item.n.01"},{"index":25,"sense_key":"point%1:06:03::","synset_id":"distributor_point.n.01"},{"index":26,"sense_key":"point%1:24:00::","synset_id":"compass_point.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"point%1:23:01::","call_ids":["semeval2013.d001.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"point%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1469,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.009535000000000002,"input_uncached_usd":0.007345000000000001,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.301977666094899}
{"item_id":"semeval2013.d001.s008.t003","gold_sense_keys":["night%1:28:03::"],"candidates":[{"index":1,"sense_key":"night%1:28:01::","synset_id":"night.n.02"},{"index":2,"sense_key":"night%1:28:00::","synset_id":"night.n.01"},{"index":3,"sense_key":"night%1:28:04::","synset_id":"night.n.03"},{"index":4,"sense_key":"night%1:28:03::","synset_id":"night.n.07"},{"index":5,"sense_key":"night%1:28:05::","synset_id":"night.n.06"},{"index":6,"sense_key":"night%1:26:00::","synset_id":"night.n.05"},{"index":7,"sense_key":"night%1:28:02::","synset_id":"night.n.04"},{"index":8,"sense_key":"night%1:18:00::","synset_id":"nox.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"night%1:28:02::","call_ids":["semeval2013.d001.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"night%1:28:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019135,"input_uncached_usd":0.003295,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.990948082995601}
{"item_id":"semeval2013.d001.s009.t000","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":2,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":3,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":4,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:04:04::","synset_id":"game.n.11"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":9,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":10,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":11,"sense_key":"game%1:23:00::","synset_id":"game.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d001.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":191,"reasoning_output_tokens":175},"cost":{"currency":"USD","total_usd":0.009865000000000002,"input_uncached_usd":0.004135000000000001,"input_cached_usd":0.0,"output_usd":0.00573,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9114670420531183}
{"item_id":"semeval2013.d001.s009.t003","gold_sense_keys":["pressure%1:07:00::"],"candidates":[{"index":1,"sense_key":"pressure%1:04:00::","synset_id":"press.n.09"},{"index":2,"sense_key":"pressure%1:19:01::","synset_id":"atmospheric_pressure.n.01"},{"index":3,"sense_key":"pressure%1:26:00::","synset_id":"imperativeness.n.01"},{"index":4,"sense_key":"pressure%1:19:00::","synset_id":"pressure.n.01"},{"index":5,"sense_key":"pressure%1:07:00::","synset_id":"pressure.n.02"},{"index":6,"sense_key":"pressure%1:09:00::","synset_id":"pressure.n.05"},{"index":7,"sense_key":"pressure%1:26:01::","synset_id":"pressure.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"pressure%1:07:00::","call_ids":["semeval2013.d001.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"pressure%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":719,"cached_input_tokens":0,"output_tokens":305,"reasoning_output_tokens":289},"cost":{"currency":"USD","total_usd":0.012745,"input_uncached_usd":0.003595,"input_cached_usd":0.0,"output_usd":0.00915,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.635494583053514}
{"item_id":"semeval2013.d001.s009.t004","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":2,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":3,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":4,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":5,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":6,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":7,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":8,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":9,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":10,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d001.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":106},"cost":{"currency":"USD","total_usd":0.007795000000000001,"input_uncached_usd":0.004135000000000001,"input_cached_usd":0.0,"output_usd":0.00366,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9767805410083383}
{"item_id":"semeval2013.d001.s010.t000","gold_sense_keys":["role%1:04:00::"],"candidates":[{"index":1,"sense_key":"role%1:07:00::","synset_id":"function.n.02"},{"index":2,"sense_key":"role%1:04:01::","synset_id":"role.n.04"},{"index":3,"sense_key":"role%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"role%1:09:00::","synset_id":"character.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"role%1:04:00::","call_ids":["semeval2013.d001.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"role%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":576,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.00609,"input_uncached_usd":0.00288,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7139448749367148}
{"item_id":"semeval2013.d001.s012.t000","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":2,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":3,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":4,"sense_key":"player%1:18:03::","synset_id":"player.n.05"},{"index":5,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d001.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.00487,"input_uncached_usd":0.00319,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.612010124954395}
{"item_id":"semeval2013.d001.s012.t001","gold_sense_keys":["couple%1:23:01::"],"candidates":[{"index":1,"sense_key":"couple%1:23:00::","synset_id":"couple.n.04"},{"index":2,"sense_key":"couple%1:17:00::","synset_id":"couple.n.05"},{"index":3,"sense_key":"couple%1:14:00::","synset_id":"couple.n.01"},{"index":4,"sense_key":"couple%1:14:01::","synset_id":"couple.n.02"},{"index":5,"sense_key":"couple%1:23:01::","synset_id":"couple.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"couple%1:23:01::","call_ids":["semeval2013.d001.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"couple%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.0056300000000000005,"input_uncached_usd":0.0032600000000000003,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9616559171117842}
{"item_id":"semeval2013.d001.s012.t002","gold_sense_keys":["play%1:04:05::"],"candidates":[{"index":1,"sense_key":"play%1:04:00::","synset_id":"play.n.14"},{"index":2,"sense_key":"play%1:26:01::","synset_id":"free_rein.n.01"},{"index":3,"sense_key":"play%1:10:02::","synset_id":"fun.n.02"},{"index":4,"sense_key":"play%1:04:07::","synset_id":"play.n.17"},{"index":5,"sense_key":"play%1:04:04::","synset_id":"maneuver.n.03"},{"index":6,"sense_key":"play%1:10:01::","synset_id":"play.n.01"},{"index":7,"sense_key":"play%1:04:05::","synset_id":"play.n.03"},{"index":8,"sense_key":"play%1:04:08::","synset_id":"play.n.06"},{"index":9,"sense_key":"play%1:07:00::","synset_id":"looseness.n.05"},{"index":10,"sense_key":"play%1:28:00::","synset_id":"playing_period.n.01"},{"index":11,"sense_key":"play%1:04:01::","synset_id":"turn.n.03"},{"index":12,"sense_key":"play%1:04:02::","synset_id":"gambling.n.01"},{"index":13,"sense_key":"play%1:26:00::","synset_id":"play.n.05"},{"index":14,"sense_key":"play%1:10:03::","synset_id":"play.n.02"},{"index":15,"sense_key":"play%1:04:06::","synset_id":"play.n.08"},{"index":16,"sense_key":"play%1:11:00::","synset_id":"shimmer.n.01"},{"index":17,"sense_key":"play%1:04:03::","synset_id":"bid.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"play%1:04:05::","call_ids":["semeval2013.d001.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"play%1:04:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1252,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00821,"input_uncached_usd":0.006260000000000001,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3972691249800846}
{"item_id":"semeval2013.d001.s012.t003","gold_sense_keys":["help%1:04:00::"],"candidates":[{"index":1,"sense_key":"help%1:07:00::","synset_id":"aid.n.01"},{"index":2,"sense_key":"help%1:07:01::","synset_id":"avail.n.01"},{"index":3,"sense_key":"help%1:04:00::","synset_id":"aid.n.02"},{"index":4,"sense_key":"help%1:18:00::","synset_id":"assistant.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"help%1:04:00::","call_ids":["semeval2013.d001.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"help%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":129},"cost":{"currency":"USD","total_usd":0.007620000000000001,"input_uncached_usd":0.0032700000000000003,"input_cached_usd":0.0,"output_usd":0.00435,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.793303541955538}
{"item_id":"semeval2013.d001.s012.t005","gold_sense_keys":["lead%1:23:00::"],"candidates":[{"index":1,"sense_key":"lead%1:07:01::","synset_id":"spark_advance.n.01"},{"index":2,"sense_key":"lead%1:04:01::","synset_id":"lead.n.04"},{"index":3,"sense_key":"lead%1:07:02::","synset_id":"lead.n.01"},{"index":4,"sense_key":"lead%1:06:01::","synset_id":"jumper_cable.n.01"},{"index":5,"sense_key":"lead%1:06:00::","synset_id":"lead.n.15"},{"index":6,"sense_key":"lead%1:15:00::","synset_id":"lead.n.09"},{"index":7,"sense_key":"lead%1:23:00::","synset_id":"lead.n.07"},{"index":8,"sense_key":"lead%1:27:00::","synset_id":"lead.n.02"},{"index":9,"sense_key":"lead%1:10:01::","synset_id":"lead.n.06"},{"index":10,"sense_key":"lead%1:10:00::","synset_id":"lead.n.11"},{"index":11,"sense_key":"lead%1:04:00::","synset_id":"lead.n.17"},{"index":12,"sense_key":"lead%1:06:03::","synset_id":"leash.n.01"},{"index":13,"sense_key":"lead%1:25:00::","synset_id":"lead.n.05"},{"index":14,"sense_key":"lead%1:10:02::","synset_id":"tip.n.03"},{"index":15,"sense_key":"lead%1:06:02::","synset_id":"lead.n.14"},{"index":16,"sense_key":"lead%1:18:00::","synset_id":"star.n.04"},{"index":17,"sense_key":"lead%1:09:00::","synset_id":"lead.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"lead%1:23:00::","call_ids":["semeval2013.d001.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"lead%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1177,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":110},"cost":{"currency":"USD","total_usd":0.009665,"input_uncached_usd":0.0058850000000000005,"input_cached_usd":0.0,"output_usd":0.00378,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7122483749408275}
{"item_id":"semeval2013.d001.s012.t006","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:01::","synset_id":"team.n.02"},{"index":2,"sense_key":"team%1:14:00::","synset_id":"team.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.004030000000000001,"input_uncached_usd":0.0026200000000000004,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3296266660327092}
{"item_id":"semeval2013.d001.s012.t007","gold_sense_keys":["end%1:09:00::","end%1:11:00::","end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":2,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":3,"sense_key":"end%1:18:00::","synset_id":"end.n.08"},{"index":4,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":5,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":6,"sense_key":"end%1:04:02::","synset_id":"end.n.14"},{"index":7,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":8,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":9,"sense_key":"end%1:15:00::","synset_id":"end.n.01"},{"index":10,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":11,"sense_key":"end%1:15:01::","synset_id":"end.n.09"},{"index":12,"sense_key":"end%1:15:02::","synset_id":"end.n.07"},{"index":13,"sense_key":"end%1:11:00::","synset_id":"end.n.03"},{"index":14,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"end%1:28:00::","call_ids":["semeval2013.d001.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"end%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1062,"cached_input_tokens":0,"output_tokens":398,"reasoning_output_tokens":382},"cost":{"currency":"USD","total_usd":0.01725,"input_uncached_usd":0.0053100000000000005,"input_cached_usd":0.0,"output_usd":0.011940000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.9869581669336185}
{"item_id":"semeval2013.d001.s012.t008","gold_sense_keys":["quarter%1:28:01::"],"candidates":[{"index":1,"sense_key":"quarter%1:18:00::","synset_id":"quarter.n.11"},{"index":2,"sense_key":"quarter%1:23:04::","synset_id":"quarter.n.09"},{"index":3,"sense_key":"quarter%1:04:00::","synset_id":"quarter.n.14"},{"index":4,"sense_key":"quarter%1:23:01::","synset_id":"one-fourth.n.01"},{"index":5,"sense_key":"quarter%1:28:02::","synset_id":"quarter.n.06"},{"index":6,"sense_key":"quarter%1:28:01::","synset_id":"quarter.n.03"},{"index":7,"sense_key":"quarter%1:06:01::","synset_id":"quarter.n.13"},{"index":8,"sense_key":"quarter%1:21:00::","synset_id":"quarter.n.10"},{"index":9,"sense_key":"quarter%1:23:02::","synset_id":"quarter.n.08"},{"index":10,"sense_key":"quarter%1:28:03::","synset_id":"quarter.n.04"},{"index":11,"sense_key":"quarter%1:15:00::","synset_id":"quarter.n.02"},{"index":12,"sense_key":"quarter%1:28:00::","synset_id":"quarter.n.05"},{"index":13,"sense_key":"quarter%1:24:00::","synset_id":"quarter.n.07"},{"index":14,"sense_key":"quarter%1:06:00::","synset_id":"stern.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"quarter%1:28:01::","call_ids":["semeval2013.d001.s012.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"quarter%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":973,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.0072050000000000005,"input_uncached_usd":0.004865,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9847715420182794}
{"item_id":"semeval2013.d001.s013.t001","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"},{"index":2,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":3,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":4,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":5,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":6,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"minute%1:28:00::","call_ids":["semeval2013.d001.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00538,"input_uncached_usd":0.0034600000000000004,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.776385791017674}
{"item_id":"semeval2013.d001.s014.t000","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:01::","synset_id":"team.n.02"},{"index":2,"sense_key":"team%1:14:00::","synset_id":"team.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.004030000000000001,"input_uncached_usd":0.00256,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.595028042094782}
{"item_id":"semeval2013.d001.s014.t001","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":43,"reasoning_output_tokens":28},"cost":{"currency":"USD","total_usd":0.003855,"input_uncached_usd":0.002565,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.1488887500017881}
{"item_id":"semeval2013.d001.s014.t002","gold_sense_keys":["line%1:15:01::"],"candidates":[{"index":1,"sense_key":"line%1:15:01::","synset_id":"line.n.20"},{"index":2,"sense_key":"line%1:06:00::","synset_id":"line.n.18"},{"index":3,"sense_key":"line%1:04:01::","synset_id":"line.n.16"},{"index":4,"sense_key":"line%1:15:00::","synset_id":"line.n.11"},{"index":5,"sense_key":"line%1:06:07::","synset_id":"telephone_line.n.02"},{"index":6,"sense_key":"line%1:06:01::","synset_id":"cable.n.02"},{"index":7,"sense_key":"line%1:10:02::","synset_id":"line.n.05"},{"index":8,"sense_key":"line%1:23:00::","synset_id":"agate_line.n.01"},{"index":9,"sense_key":"line%1:10:01::","synset_id":"line.n.02"},{"index":10,"sense_key":"line%1:04:00::","synset_id":"occupation.n.01"},{"index":11,"sense_key":"line%1:10:05::","synset_id":"channel.n.05"},{"index":12,"sense_key":"line%1:09:01::","synset_id":"argumentation.n.02"},{"index":13,"sense_key":"line%1:21:00::","synset_id":"credit_line.n.01"},{"index":14,"sense_key":"line%1:06:05::","synset_id":"line.n.23"},{"index":15,"sense_key":"line%1:06:03::","synset_id":"line.n.14"},{"index":16,"sense_key":"line%1:25:01::","synset_id":"wrinkle.n.01"},{"index":17,"sense_key":"line%1:19:00::","synset_id":"line.n.06"},{"index":18,"sense_key":"line%1:10:04::","synset_id":"note.n.02"},{"index":19,"sense_key":"line%1:09:00::","synset_id":"line.n.29"},{"index":20,"sense_key":"line%1:14:00::","synset_id":"lineage.n.01"},{"index":21,"sense_key":"line%1:06:08::","synset_id":"line.n.22"},{"index":22,"sense_key":"line%1:14:02::","synset_id":"course.n.02"},{"index":23,"sense_key":"line%1:10:03::","synset_id":"tune.n.01"},{"index":24,"sense_key":"line%1:06:06::","synset_id":"pipeline.n.02"},{"index":25,"sense_key":"line%1:14:01::","synset_id":"line.n.03"},{"index":26,"sense_key":"line%1:25:00::","synset_id":"line.n.04"},{"index":27,"sense_key":"line%1:14:03::","synset_id":"line.n.01"},{"index":28,"sense_key":"line%1:10:06::","synset_id":"line.n.27"},{"index":29,"sense_key":"line%1:06:09::","synset_id":"production_line.n.01"},{"index":30,"sense_key":"line%1:15:02::","synset_id":"line.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"line%1:15:01::","call_ids":["semeval2013.d001.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"line%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1704,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.010740000000000001,"input_uncached_usd":0.008520000000000002,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.797398000024259}
{"item_id":"semeval2013.d001.s015.t001","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:01::","synset_id":"team.n.02"},{"index":2,"sense_key":"team%1:14:00::","synset_id":"team.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.0036300000000000004,"input_uncached_usd":0.0022800000000000003,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2078332089586183}
{"item_id":"semeval2013.d001.s015.t003","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":2,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":3,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":4,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":5,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":8,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":9,"sense_key":"game%1:04:04::","synset_id":"game.n.11"},{"index":10,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":11,"sense_key":"game%1:06:00::","synset_id":"game.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d001.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":824,"cached_input_tokens":0,"output_tokens":211,"reasoning_output_tokens":195},"cost":{"currency":"USD","total_usd":0.010450000000000001,"input_uncached_usd":0.00412,"input_cached_usd":0.0,"output_usd":0.0063300000000000006,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.483784458017908}
{"item_id":"semeval2013.d001.s015.t005","gold_sense_keys":["time%1:11:00::"],"candidates":[{"index":1,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":2,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":3,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":4,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":5,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":6,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":7,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":8,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":9,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"time%1:11:00::","call_ids":["semeval2013.d001.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"time%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":805,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.0056749999999999995,"input_uncached_usd":0.004025,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7806532919639722}
{"item_id":"semeval2013.d001.s015.t006","gold_sense_keys":["season%1:28:02::"],"candidates":[{"index":1,"sense_key":"season%1:28:02::","synset_id":"season.n.01"},{"index":2,"sense_key":"season%1:28:00::","synset_id":"season.n.02"},{"index":3,"sense_key":"season%1:28:01::","synset_id":"season.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"season%1:28:02::","call_ids":["semeval2013.d001.s015.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"season%1:28:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.00536,"input_uncached_usd":0.00269,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8856581250438467}
{"item_id":"semeval2013.d002.s000.t000","gold_sense_keys":["wall_street%1:14:00::"],"candidates":[{"index":1,"sense_key":"wall_street%1:15:00::","synset_id":"wall_street.n.01"},{"index":2,"sense_key":"wall_street%1:14:00::","synset_id":"wall_street.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"wall_street%1:14:00::","call_ids":["semeval2013.d002.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"wall_street%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":288,"cached_input_tokens":0,"output_tokens":266,"reasoning_output_tokens":251},"cost":{"currency":"USD","total_usd":0.009420000000000001,"input_uncached_usd":0.00144,"input_cached_usd":0.0,"output_usd":0.007980000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.872814500005916}
{"item_id":"semeval2013.d002.s000.t002","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":2,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"},{"index":3,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"},{"index":4,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d002.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":362,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.0038800000000000006,"input_uncached_usd":0.0018100000000000002,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4892963339807466}
{"item_id":"semeval2013.d002.s001.t001","gold_sense_keys":["heading%1:15:00::"],"candidates":[{"index":1,"sense_key":"heading%1:15:00::","synset_id":"bearing.n.02"},{"index":2,"sense_key":"heading%1:06:00::","synset_id":"drift.n.07"},{"index":3,"sense_key":"heading%1:10:00::","synset_id":"heading.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"heading%1:15:00::","call_ids":["semeval2013.d002.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"heading%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.00384,"input_uncached_usd":0.00198,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.950130125042051}
{"item_id":"semeval2013.d002.s001.t003","gold_sense_keys":["market%1:14:01::"],"candidates":[{"index":1,"sense_key":"market%1:14:00::","synset_id":"market.n.02"},{"index":2,"sense_key":"market%1:04:00::","synset_id":"market.n.01"},{"index":3,"sense_key":"market%1:06:00::","synset_id":"grocery_store.n.01"},{"index":4,"sense_key":"market%1:06:01::","synset_id":"marketplace.n.02"},{"index":5,"sense_key":"market%1:14:01::","synset_id":"market.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"market%1:14:01::","call_ids":["semeval2013.d002.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"market%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.004175,"input_uncached_usd":0.002405,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6462053749710321}
{"item_id":"semeval2013.d002.s001.t004","gold_sense_keys":["indicator%1:10:00::"],"candidates":[{"index":1,"sense_key":"indicator%1:06:00::","synset_id":"indicator.n.03"},{"index":2,"sense_key":"indicator%1:10:00::","synset_id":"index.n.02"},{"index":3,"sense_key":"indicator%1:27:00::","synset_id":"indicator.n.04"},{"index":4,"sense_key":"indicator%1:10:01::","synset_id":"indicator.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"indicator%1:10:00::","call_ids":["semeval2013.d002.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"indicator%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00404,"input_uncached_usd":0.0021200000000000004,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5955242499476299}
{"item_id":"semeval2013.d002.s001.t005","gold_sense_keys":["united_states%1:15:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"united_states%1:15:00::","call_ids":["semeval2013.d002.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"united_states%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.004035,"input_uncached_usd":0.001935,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.637455250020139}
{"item_id":"semeval2013.d002.s001.t006","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":2,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"},{"index":3,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"},{"index":4,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d002.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.00409,"input_uncached_usd":0.0020800000000000003,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9186119999503717}
{"item_id":"semeval2013.d002.s002.t001","gold_sense_keys":["figure%1:07:01::"],"candidates":[{"index":1,"sense_key":"figure%1:06:01::","synset_id":"design.n.04"},{"index":2,"sense_key":"figure%1:04:00::","synset_id":"figure.n.13"},{"index":3,"sense_key":"figure%1:10:00::","synset_id":"figure.n.01"},{"index":4,"sense_key":"figure%1:07:01::","synset_id":"number.n.01"},{"index":5,"sense_key":"figure%1:21:00::","synset_id":"figure.n.07"},{"index":6,"sense_key":"figure%1:08:00::","synset_id":"human_body.n.01"},{"index":7,"sense_key":"figure%1:18:00::","synset_id":"name.n.04"},{"index":8,"sense_key":"figure%1:10:01::","synset_id":"trope.n.01"},{"index":9,"sense_key":"figure%1:07:00::","synset_id":"figure.n.08"},{"index":10,"sense_key":"figure%1:23:00::","synset_id":"digit.n.01"},{"index":11,"sense_key":"figure%1:25:00::","synset_id":"figure.n.06"},{"index":12,"sense_key":"figure%1:06:00::","synset_id":"figure.n.04"},{"index":13,"sense_key":"figure%1:09:00::","synset_id":"figure.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"figure%1:07:01::","call_ids":["semeval2013.d002.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"figure%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":828,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":131},"cost":{"currency":"USD","total_usd":0.008550000000000002,"input_uncached_usd":0.0041400000000000005,"input_cached_usd":0.0,"output_usd":0.00441,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8422513749683276}
{"item_id":"semeval2013.d002.s002.t006","gold_sense_keys":["predominance%1:26:00::"],"candidates":[{"index":1,"sense_key":"predominance%1:07:00::","synset_id":"predomination.n.02"},{"index":2,"sense_key":"predominance%1:26:00::","synset_id":"predominance.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"predominance%1:26:00::","call_ids":["semeval2013.d002.s002.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"predominance%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":382,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":121},"cost":{"currency":"USD","total_usd":0.00602,"input_uncached_usd":0.0019100000000000002,"input_cached_usd":0.0,"output_usd":0.00411,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.829043166944757}
{"item_id":"semeval2013.d002.s004.t000","gold_sense_keys":["market%1:14:01::"],"candidates":[{"index":1,"sense_key":"market%1:14:00::","synset_id":"market.n.02"},{"index":2,"sense_key":"market%1:06:01::","synset_id":"marketplace.n.02"},{"index":3,"sense_key":"market%1:14:01::","synset_id":"market.n.04"},{"index":4,"sense_key":"market%1:04:00::","synset_id":"market.n.01"},{"index":5,"sense_key":"market%1:06:00::","synset_id":"grocery_store.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"market%1:14:01::","call_ids":["semeval2013.d002.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"market%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0047150000000000004,"input_uncached_usd":0.0028250000000000003,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4430956669384614}
{"item_id":"semeval2013.d002.s005.t002","gold_sense_keys":["volume%1:07:03::"],"candidates":[{"index":1,"sense_key":"volume%1:23:01::","synset_id":"volume.n.05"},{"index":2,"sense_key":"volume%1:06:00::","synset_id":"book.n.02"},{"index":3,"sense_key":"volume%1:07:03::","synset_id":"bulk.n.02"},{"index":4,"sense_key":"volume%1:10:00::","synset_id":"volume.n.04"},{"index":5,"sense_key":"volume%1:07:02::","synset_id":"volume.n.06"},{"index":6,"sense_key":"volume%1:23:00::","synset_id":"volume.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"volume%1:07:03::","call_ids":["semeval2013.d002.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"volume%1:07:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019085,"input_uncached_usd":0.003245,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.375824957969598}
{"item_id":"semeval2013.d002.s005.t003","gold_sense_keys":["case%1:09:00::"],"candidates":[{"index":1,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":4,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:06:01::","synset_id":"case.n.20"},{"index":7,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":8,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":9,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":10,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":11,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":12,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":13,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":14,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":17,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":20,"sense_key":"case%1:10:03::","synset_id":"font.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"case%1:09:00::","call_ids":["semeval2013.d002.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"case%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1250,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":80},"cost":{"currency":"USD","total_usd":0.0091,"input_uncached_usd":0.00625,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.242045541992411}
{"item_id":"semeval2013.d002.s005.t004","gold_sense_keys":["end%1:09:00::","end%1:11:00::","end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":2,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":3,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":4,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":5,"sense_key":"end%1:18:00::","synset_id":"end.n.08"},{"index":6,"sense_key":"end%1:15:01::","synset_id":"end.n.09"},{"index":7,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":8,"sense_key":"end%1:15:00::","synset_id":"end.n.01"},{"index":9,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":10,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"},{"index":11,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":12,"sense_key":"end%1:11:00::","synset_id":"end.n.03"},{"index":13,"sense_key":"end%1:04:02::","synset_id":"end.n.14"},{"index":14,"sense_key":"end%1:15:02::","synset_id":"end.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"end%1:28:00::","call_ids":["semeval2013.d002.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"end%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":982,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.00689,"input_uncached_usd":0.00491,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3970975830452517}
{"item_id":"semeval2013.d002.s005.t005","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:14:00::","synset_id":"class.n.06"},{"index":3,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":4,"sense_key":"year%1:28:00::","synset_id":"year.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d002.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.005075,"input_uncached_usd":0.002915,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6699526669690385}
{"item_id":"semeval2013.d002.s006.t000","gold_sense_keys":["trend%1:15:02::"],"candidates":[{"index":1,"sense_key":"trend%1:09:02::","synset_id":"vogue.n.01"},{"index":2,"sense_key":"trend%1:15:02::","synset_id":"tendency.n.04"},{"index":3,"sense_key":"trend%1:15:01::","synset_id":"course.n.03"},{"index":4,"sense_key":"trend%1:09:00::","synset_id":"drift.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"trend%1:15:02::","call_ids":["semeval2013.d002.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"trend%1:15:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":117},"cost":{"currency":"USD","total_usd":0.006975,"input_uncached_usd":0.0029850000000000002,"input_cached_usd":0.0,"output_usd":0.0039900000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8034423750359565}
{"item_id":"semeval2013.d002.s006.t001","gold_sense_keys":["publication%1:04:02::"],"candidates":[{"index":1,"sense_key":"publication%1:04:02::","synset_id":"publication.n.03"},{"index":2,"sense_key":"publication%1:04:00::","synset_id":"publication.n.04"},{"index":3,"sense_key":"publication%1:04:01::","synset_id":"issue.n.11"},{"index":4,"sense_key":"publication%1:10:00::","synset_id":"publication.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"publication%1:04:02::","call_ids":["semeval2013.d002.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"publication%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":94},"cost":{"currency":"USD","total_usd":0.00588,"input_uncached_usd":0.0026100000000000003,"input_cached_usd":0.0,"output_usd":0.00327,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3752996659604833}
{"item_id":"semeval2013.d002.s006.t002","gold_sense_keys":["indicator%1:10:00::"],"candidates":[{"index":1,"sense_key":"indicator%1:10:00::","synset_id":"index.n.02"},{"index":2,"sense_key":"indicator%1:06:00::","synset_id":"indicator.n.03"},{"index":3,"sense_key":"indicator%1:27:00::","synset_id":"indicator.n.04"},{"index":4,"sense_key":"indicator%1:10:01::","synset_id":"indicator.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"indicator%1:10:00::","call_ids":["semeval2013.d002.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"indicator%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.00482,"input_uncached_usd":0.00284,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.497658957960084}
{"item_id":"semeval2013.d002.s006.t003","gold_sense_keys":["market%1:14:01::"],"candidates":[{"index":1,"sense_key":"market%1:06:01::","synset_id":"marketplace.n.02"},{"index":2,"sense_key":"market%1:14:00::","synset_id":"market.n.02"},{"index":3,"sense_key":"market%1:14:01::","synset_id":"market.n.04"},{"index":4,"sense_key":"market%1:04:00::","synset_id":"market.n.01"},{"index":5,"sense_key":"market%1:06:00::","synset_id":"grocery_store.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"market%1:14:01::","call_ids":["semeval2013.d002.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"market%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":132},"cost":{"currency":"USD","total_usd":0.00757,"input_uncached_usd":0.0031300000000000004,"input_cached_usd":0.0,"output_usd":0.00444,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6668137919623405}
{"item_id":"semeval2013.d002.s006.t005","gold_sense_keys":["relation%1:03:00::"],"candidates":[{"index":1,"sense_key":"relation%1:04:01::","synset_id":"sexual_intercourse.n.01"},{"index":2,"sense_key":"relation%1:09:00::","synset_id":"relation_back.n.01"},{"index":3,"sense_key":"relation%1:04:00::","synset_id":"relation.n.06"},{"index":4,"sense_key":"relation%1:18:00::","synset_id":"relative.n.01"},{"index":5,"sense_key":"relation%1:10:01::","synset_id":"relation.n.04"},{"index":6,"sense_key":"relation%1:03:00::","synset_id":"relation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"relation%1:03:00::","call_ids":["semeval2013.d002.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"relation%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":103},"cost":{"currency":"USD","total_usd":0.0070799999999999995,"input_uncached_usd":0.00351,"input_cached_usd":0.0,"output_usd":0.0035700000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.496502958936617}
{"item_id":"semeval2013.d002.s006.t007","gold_sense_keys":["recovery%1:11:00::"],"candidates":[{"index":1,"sense_key":"recovery%1:04:00::","synset_id":"recovery.n.03"},{"index":2,"sense_key":"recovery%1:11:00::","synset_id":"recovery.n.01"},{"index":3,"sense_key":"recovery%1:22:00::","synset_id":"convalescence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"recovery%1:11:00::","call_ids":["semeval2013.d002.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"recovery%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":159,"reasoning_output_tokens":143},"cost":{"currency":"USD","total_usd":0.007375000000000001,"input_uncached_usd":0.002605,"input_cached_usd":0.0,"output_usd":0.00477,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.138450833968818}
{"item_id":"semeval2013.d002.s006.t008","gold_sense_keys":["analyst%1:18:00::","analyst%1:18:02::"],"candidates":[{"index":1,"sense_key":"analyst%1:18:00::","synset_id":"analyst.n.01"},{"index":2,"sense_key":"analyst%1:18:01::","synset_id":"analyst.n.03"},{"index":3,"sense_key":"analyst%1:18:02::","synset_id":"analyst.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"analyst%1:18:02::","call_ids":["semeval2013.d002.s006.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"analyst%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.003965,"input_uncached_usd":0.0025250000000000003,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.505115665961057}
{"item_id":"semeval2013.d002.s007.t000","gold_sense_keys":["united_states%1:15:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"united_states%1:15:00::","call_ids":["semeval2013.d002.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"united_states%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.004450000000000001,"input_uncached_usd":0.0025900000000000003,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9085425829980522}
{"item_id":"semeval2013.d002.s007.t003","gold_sense_keys":["month%1:28:01::"],"candidates":[{"index":1,"sense_key":"month%1:28:00::","synset_id":"month.n.02"},{"index":2,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:01::","call_ids":["semeval2013.d002.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.00545,"input_uncached_usd":0.00236,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.201847624965012}
{"item_id":"semeval2013.d002.s007.t004","gold_sense_keys":["row%1:07:00::"],"candidates":[{"index":1,"sense_key":"row%1:04:00::","synset_id":"rowing.n.01"},{"index":2,"sense_key":"row%1:17:00::","synset_id":"row.n.03"},{"index":3,"sense_key":"row%1:14:01::","synset_id":"row.n.05"},{"index":4,"sense_key":"row%1:10:00::","synset_id":"quarrel.n.01"},{"index":5,"sense_key":"row%1:06:00::","synset_id":"course.n.08"},{"index":6,"sense_key":"row%1:14:00::","synset_id":"row.n.01"},{"index":7,"sense_key":"row%1:07:00::","synset_id":"row.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"row%1:07:00::","call_ids":["semeval2013.d002.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"row%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.004955,"input_uncached_usd":0.0032150000000000004,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6501608749385923}
{"item_id":"semeval2013.d002.s008.t000","gold_sense_keys":["confidence%1:12:00::","confidence%1:26:02::"],"candidates":[{"index":1,"sense_key":"confidence%1:09:00::","synset_id":"assurance.n.01"},{"index":2,"sense_key":"confidence%1:12:00::","synset_id":"confidence.n.02"},{"index":3,"sense_key":"confidence%1:26:02::","synset_id":"confidence.n.03"},{"index":4,"sense_key":"confidence%1:26:03::","synset_id":"confidence.n.04"},{"index":5,"sense_key":"confidence%1:10:00::","synset_id":"confidence.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"confidence%1:26:02::","call_ids":["semeval2013.d002.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"confidence%1:26:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.0058200000000000005,"input_uncached_usd":0.0030900000000000003,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6225487909978256}
{"item_id":"semeval2013.d002.s008.t003","gold_sense_keys":["month%1:28:01::"],"candidates":[{"index":1,"sense_key":"month%1:28:00::","synset_id":"month.n.02"},{"index":2,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:01::","call_ids":["semeval2013.d002.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":466,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.00452,"input_uncached_usd":0.00233,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8780106670456007}
{"item_id":"semeval2013.d002.s008.t004","gold_sense_keys":["estimate%1:09:00::"],"candidates":[{"index":1,"sense_key":"estimate%1:09:01::","synset_id":"estimate.n.05"},{"index":2,"sense_key":"estimate%1:10:00::","synset_id":"appraisal.n.02"},{"index":3,"sense_key":"estimate%1:04:01::","synset_id":"estimate.n.02"},{"index":4,"sense_key":"estimate%1:10:01::","synset_id":"estimate.n.04"},{"index":5,"sense_key":"estimate%1:09:00::","synset_id":"estimate.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"estimate%1:09:00::","call_ids":["semeval2013.d002.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"estimate%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.0055850000000000006,"input_uncached_usd":0.0030650000000000004,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8808727089781314}
{"item_id":"semeval2013.d002.s008.t005","gold_sense_keys":["index%1:10:01::"],"candidates":[{"index":1,"sense_key":"index%1:08:00::","synset_id":"index.n.05"},{"index":2,"sense_key":"index%1:10:00::","synset_id":"index.n.04"},{"index":3,"sense_key":"index%1:10:01::","synset_id":"index.n.02"},{"index":4,"sense_key":"index%1:24:00::","synset_id":"index.n.01"},{"index":5,"sense_key":"index%1:10:02::","synset_id":"exponent.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"index%1:10:01::","call_ids":["semeval2013.d002.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"index%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01869,"input_uncached_usd":0.00285,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.778258792008273}
{"item_id":"semeval2013.d002.s009.t000","gold_sense_keys":["index%1:10:01::"],"candidates":[{"index":1,"sense_key":"index%1:08:00::","synset_id":"index.n.05"},{"index":2,"sense_key":"index%1:24:00::","synset_id":"index.n.01"},{"index":3,"sense_key":"index%1:10:00::","synset_id":"index.n.04"},{"index":4,"sense_key":"index%1:10:01::","synset_id":"index.n.02"},{"index":5,"sense_key":"index%1:10:02::","synset_id":"exponent.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"index%1:10:01::","call_ids":["semeval2013.d002.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"index%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":828,"reasoning_output_tokens":806},"cost":{"currency":"USD","total_usd":0.027565000000000003,"input_uncached_usd":0.0027250000000000004,"input_cached_usd":0.0,"output_usd":0.02484,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":24.661748167010956}
{"item_id":"semeval2013.d002.s009.t001","gold_sense_keys":["rise%1:04:01::","rise%1:11:00::"],"candidates":[{"index":1,"sense_key":"rise%1:07:03::","synset_id":"rise.n.09"},{"index":2,"sense_key":"rise%1:04:01::","synset_id":"advance.n.06"},{"index":3,"sense_key":"rise%1:04:00::","synset_id":"rise.n.02"},{"index":4,"sense_key":"rise%1:07:02::","synset_id":"raise.n.01"},{"index":5,"sense_key":"rise%1:17:00::","synset_id":"ascent.n.01"},{"index":6,"sense_key":"rise%1:11:02::","synset_id":"rise.n.04"},{"index":7,"sense_key":"rise%1:11:01::","synset_id":"lift.n.04"},{"index":8,"sense_key":"rise%1:11:03::","synset_id":"emanation.n.03"},{"index":9,"sense_key":"rise%1:07:01::","synset_id":"upgrade.n.04"},{"index":10,"sense_key":"rise%1:11:00::","synset_id":"rise.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"rise%1:04:01::","call_ids":["semeval2013.d002.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"rise%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.0066,"input_uncached_usd":0.00387,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.881460874923505}
{"item_id":"semeval2013.d002.s009.t002","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"},{"index":2,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"},{"index":3,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":4,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d002.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.004535,"input_uncached_usd":0.002585,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.059293832979165}
{"item_id":"semeval2013.d002.s009.t003","gold_sense_keys":["high%1:07:00::"],"candidates":[{"index":1,"sense_key":"high%1:26:01::","synset_id":"high.n.04"},{"index":2,"sense_key":"high%1:14:00::","synset_id":"senior_high_school.n.01"},{"index":3,"sense_key":"high%1:07:00::","synset_id":"high.n.01"},{"index":4,"sense_key":"high%1:15:00::","synset_id":"high.n.05"},{"index":5,"sense_key":"high%1:06:00::","synset_id":"high_gear.n.01"},{"index":6,"sense_key":"high%1:26:00::","synset_id":"high.n.02"},{"index":7,"sense_key":"high%1:26:02::","synset_id":"high.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"high%1:07:00::","call_ids":["semeval2013.d002.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"high%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.00527,"input_uncached_usd":0.00317,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1380379159236327}
{"item_id":"semeval2013.d002.s009.t005","gold_sense_keys":["pressure%1:07:00::"],"candidates":[{"index":1,"sense_key":"pressure%1:09:00::","synset_id":"pressure.n.05"},{"index":2,"sense_key":"pressure%1:07:00::","synset_id":"pressure.n.02"},{"index":3,"sense_key":"pressure%1:04:00::","synset_id":"press.n.09"},{"index":4,"sense_key":"pressure%1:26:00::","synset_id":"imperativeness.n.01"},{"index":5,"sense_key":"pressure%1:26:01::","synset_id":"pressure.n.06"},{"index":6,"sense_key":"pressure%1:19:01::","synset_id":"atmospheric_pressure.n.01"},{"index":7,"sense_key":"pressure%1:19:00::","synset_id":"pressure.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pressure%1:07:00::","call_ids":["semeval2013.d002.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pressure%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":135},"cost":{"currency":"USD","total_usd":0.00793,"input_uncached_usd":0.0034000000000000002,"input_cached_usd":0.0,"output_usd":0.00453,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8172927079722285}
{"item_id":"semeval2013.d002.s009.t006","gold_sense_keys":["sector%1:14:00::"],"candidates":[{"index":1,"sense_key":"sector%1:06:00::","synset_id":"sector.n.06"},{"index":2,"sense_key":"sector%1:14:00::","synset_id":"sector.n.02"},{"index":3,"sense_key":"sector%1:23:00::","synset_id":"sector.n.04"},{"index":4,"sense_key":"sector%1:25:00::","synset_id":"sector.n.01"},{"index":5,"sense_key":"sector%1:15:00::","synset_id":"sector.n.05"},{"index":6,"sense_key":"sector%1:09:00::","synset_id":"sector.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sector%1:14:00::","call_ids":["semeval2013.d002.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sector%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":124},"cost":{"currency":"USD","total_usd":0.007015,"input_uncached_usd":0.0028150000000000002,"input_cached_usd":0.0,"output_usd":0.0042,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8140217910986394}
{"item_id":"semeval2013.d002.s009.t007","gold_sense_keys":["situation%1:26:00::"],"candidates":[{"index":1,"sense_key":"situation%1:26:01::","synset_id":"situation.n.02"},{"index":2,"sense_key":"situation%1:04:00::","synset_id":"position.n.06"},{"index":3,"sense_key":"situation%1:15:00::","synset_id":"site.n.02"},{"index":4,"sense_key":"situation%1:26:02::","synset_id":"situation.n.03"},{"index":5,"sense_key":"situation%1:26:00::","synset_id":"situation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"situation%1:26:00::","call_ids":["semeval2013.d002.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"situation%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.0055650000000000005,"input_uncached_usd":0.0030450000000000004,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.915928625036031}
{"item_id":"semeval2013.d002.s010.t001","gold_sense_keys":["security%1:21:04::"],"candidates":[{"index":1,"sense_key":"security%1:14:00::","synset_id":"security.n.06"},{"index":2,"sense_key":"security%1:21:00::","synset_id":"security.n.05"},{"index":3,"sense_key":"security%1:06:00::","synset_id":"security_system.n.02"},{"index":4,"sense_key":"security%1:21:04::","synset_id":"security.n.04"},{"index":5,"sense_key":"security%1:21:01::","synset_id":"security.n.02"},{"index":6,"sense_key":"security%1:12:00::","synset_id":"security.n.03"},{"index":7,"sense_key":"security%1:26:00::","synset_id":"security.n.01"},{"index":8,"sense_key":"security%1:04:00::","synset_id":"security.n.09"},{"index":9,"sense_key":"security%1:10:00::","synset_id":"security.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"security%1:21:04::","call_ids":["semeval2013.d002.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"security%1:21:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.006025000000000001,"input_uncached_usd":0.0037150000000000004,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.851198416086845}
{"item_id":"semeval2013.d002.s010.t002","gold_sense_keys":["ground%1:15:00::"],"candidates":[{"index":1,"sense_key":"ground%1:17:00::","synset_id":"land.n.04"},{"index":2,"sense_key":"ground%1:16:00::","synset_id":"reason.n.01"},{"index":3,"sense_key":"ground%1:15:00::","synset_id":"ground.n.05"},{"index":4,"sense_key":"ground%1:06:01::","synset_id":"ground.n.10"},{"index":5,"sense_key":"ground%1:27:00::","synset_id":"earth.n.02"},{"index":6,"sense_key":"ground%1:17:01::","synset_id":"land.n.02"},{"index":7,"sense_key":"ground%1:06:00::","synset_id":"flat_coat.n.01"},{"index":8,"sense_key":"ground%1:09:00::","synset_id":"background.n.02"},{"index":9,"sense_key":"ground%1:06:02::","synset_id":"ground.n.09"},{"index":10,"sense_key":"ground%1:09:01::","synset_id":"ground.n.08"},{"index":11,"sense_key":"ground%1:24:00::","synset_id":"footing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ground%1:15:00::","call_ids":["semeval2013.d002.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ground%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":871,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.007475000000000001,"input_uncached_usd":0.004355,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5513171249767765}
{"item_id":"semeval2013.d002.s012.t001","gold_sense_keys":["bond%1:21:02::"],"candidates":[{"index":1,"sense_key":"bond%1:19:00::","synset_id":"chemical_bond.n.01"},{"index":2,"sense_key":"bond%1:06:01::","synset_id":"attachment.n.04"},{"index":3,"sense_key":"bond%1:27:00::","synset_id":"bond.n.07"},{"index":4,"sense_key":"bond%1:06:00::","synset_id":"shackle.n.01"},{"index":5,"sense_key":"bond%1:21:02::","synset_id":"bond.n.02"},{"index":6,"sense_key":"bond%1:07:00::","synset_id":"adhesiveness.n.01"},{"index":7,"sense_key":"bond%1:24:00::","synset_id":"alliance.n.02"},{"index":8,"sense_key":"bond%1:18:01::","synset_id":"bond.n.09"},{"index":9,"sense_key":"bond%1:18:00::","synset_id":"bond.n.08"},{"index":10,"sense_key":"bond%1:21:03::","synset_id":"bail.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"bond%1:21:02::","call_ids":["semeval2013.d002.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"bond%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.005515,"input_uncached_usd":0.004045,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3736884170211852}
{"item_id":"semeval2013.d002.s012.t002","gold_sense_keys":["yield%1:21:00::"],"candidates":[{"index":1,"sense_key":"yield%1:04:00::","synset_id":"output.n.02"},{"index":2,"sense_key":"yield%1:06:00::","synset_id":"yield.n.03"},{"index":3,"sense_key":"yield%1:23:00::","synset_id":"output.n.04"},{"index":4,"sense_key":"yield%1:21:00::","synset_id":"return.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"yield%1:21:00::","call_ids":["semeval2013.d002.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"yield%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.0044150000000000005,"input_uncached_usd":0.002465,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5399240419501439}
{"item_id":"semeval2013.d002.s012.t004","gold_sense_keys":["evening%1:28:00::"],"candidates":[{"index":1,"sense_key":"evening%1:28:02::","synset_id":"evening.n.03"},{"index":2,"sense_key":"evening%1:28:00::","synset_id":"evening.n.01"},{"index":3,"sense_key":"evening%1:28:01::","synset_id":"evening.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"evening%1:28:00::","call_ids":["semeval2013.d002.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"evening%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":132},"cost":{"currency":"USD","total_usd":0.006740000000000001,"input_uncached_usd":0.0023000000000000004,"input_cached_usd":0.0,"output_usd":0.00444,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.58278516610153}
{"item_id":"semeval2013.d002.s012.t005","gold_sense_keys":["bond%1:21:02::"],"candidates":[{"index":1,"sense_key":"bond%1:24:00::","synset_id":"alliance.n.02"},{"index":2,"sense_key":"bond%1:21:03::","synset_id":"bail.n.01"},{"index":3,"sense_key":"bond%1:27:00::","synset_id":"bond.n.07"},{"index":4,"sense_key":"bond%1:07:00::","synset_id":"adhesiveness.n.01"},{"index":5,"sense_key":"bond%1:19:00::","synset_id":"chemical_bond.n.01"},{"index":6,"sense_key":"bond%1:18:00::","synset_id":"bond.n.08"},{"index":7,"sense_key":"bond%1:18:01::","synset_id":"bond.n.09"},{"index":8,"sense_key":"bond%1:06:01::","synset_id":"attachment.n.04"},{"index":9,"sense_key":"bond%1:06:00::","synset_id":"shackle.n.01"},{"index":10,"sense_key":"bond%1:21:02::","synset_id":"bond.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"bond%1:21:02::","call_ids":["semeval2013.d002.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"bond%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.005725,"input_uncached_usd":0.004045,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7225011669797823}
{"item_id":"semeval2013.d002.s012.t006","gold_sense_keys":["yield%1:21:00::"],"candidates":[{"index":1,"sense_key":"yield%1:23:00::","synset_id":"output.n.04"},{"index":2,"sense_key":"yield%1:04:00::","synset_id":"output.n.02"},{"index":3,"sense_key":"yield%1:06:00::","synset_id":"yield.n.03"},{"index":4,"sense_key":"yield%1:21:00::","synset_id":"return.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"yield%1:21:00::","call_ids":["semeval2013.d002.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"yield%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.004355,"input_uncached_usd":0.002465,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6828359590144828}
{"item_id":"semeval2013.d002.s012.t007","gold_sense_keys":["evening%1:28:00::"],"candidates":[{"index":1,"sense_key":"evening%1:28:01::","synset_id":"evening.n.02"},{"index":2,"sense_key":"evening%1:28:02::","synset_id":"evening.n.03"},{"index":3,"sense_key":"evening%1:28:00::","synset_id":"evening.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"evening%1:28:00::","call_ids":["semeval2013.d002.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"evening%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":117},"cost":{"currency":"USD","total_usd":0.006265,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.00396,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.180182750103995}
{"item_id":"semeval2013.d003.s000.t000","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":2,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":3,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":4,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"},{"index":5,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":8,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":9,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d003.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":210,"reasoning_output_tokens":194},"cost":{"currency":"USD","total_usd":0.009230000000000002,"input_uncached_usd":0.0029300000000000003,"input_cached_usd":0.0,"output_usd":0.0063,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.647211499977857}
{"item_id":"semeval2013.d003.s001.t000","gold_sense_keys":["concern%1:12:01::"],"candidates":[{"index":1,"sense_key":"concern%1:09:00::","synset_id":"concern.n.01"},{"index":2,"sense_key":"concern%1:12:01::","synset_id":"concern.n.02"},{"index":3,"sense_key":"concern%1:09:01::","synset_id":"concern.n.04"},{"index":4,"sense_key":"concern%1:12:00::","synset_id":"concern.n.03"},{"index":5,"sense_key":"concern%1:14:00::","synset_id":"business.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"concern%1:12:01::","call_ids":["semeval2013.d003.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"concern%1:12:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018205,"input_uncached_usd":0.0023650000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":493.1427767080022}
{"item_id":"semeval2013.d003.s001.t001","gold_sense_keys":["violence%1:26:00::"],"candidates":[{"index":1,"sense_key":"violence%1:26:00::","synset_id":"violence.n.03"},{"index":2,"sense_key":"violence%1:07:00::","synset_id":"ferocity.n.01"},{"index":3,"sense_key":"violence%1:04:01::","synset_id":"violence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"violence%1:26:00::","call_ids":["semeval2013.d003.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"violence%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":377,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.005005,"input_uncached_usd":0.0018850000000000002,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3462463329778984}
{"item_id":"semeval2013.d003.s001.t002","gold_sense_keys":["instability%1:07:00::","instability%1:07:01::","instability%1:26:00::"],"candidates":[{"index":1,"sense_key":"instability%1:07:00::","synset_id":"instability.n.04"},{"index":2,"sense_key":"instability%1:07:01::","synset_id":"instability.n.02"},{"index":3,"sense_key":"instability%1:26:00::","synset_id":"instability.n.01"},{"index":4,"sense_key":"instability%1:26:01::","synset_id":"imbalance.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"instability%1:26:00::","call_ids":["semeval2013.d003.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"instability%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":359,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":114},"cost":{"currency":"USD","total_usd":0.005695,"input_uncached_usd":0.0017950000000000002,"input_cached_usd":0.0,"output_usd":0.0039000000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4013957089046016}
{"item_id":"semeval2013.d003.s001.t003","gold_sense_keys":["government%1:14:00::"],"candidates":[{"index":1,"sense_key":"government%1:09:01::","synset_id":"politics.n.02"},{"index":2,"sense_key":"government%1:09:00::","synset_id":"government.n.03"},{"index":3,"sense_key":"government%1:14:00::","synset_id":"government.n.01"},{"index":4,"sense_key":"government%1:04:00::","synset_id":"government.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"government%1:14:00::","call_ids":["semeval2013.d003.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"government%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.0035000000000000005,"input_uncached_usd":0.0020900000000000003,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3461951250210404}
{"item_id":"semeval2013.d003.s001.t006","gold_sense_keys":["auction%1:04:00::"],"candidates":[{"index":1,"sense_key":"auction%1:04:00::","synset_id":"auction.n.02"},{"index":2,"sense_key":"auction%1:04:01::","synset_id":"auction.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"auction%1:04:00::","call_ids":["semeval2013.d003.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"auction%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":318,"cached_input_tokens":0,"output_tokens":39,"reasoning_output_tokens":24},"cost":{"currency":"USD","total_usd":0.0027600000000000003,"input_uncached_usd":0.00159,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2769407089799643}
{"item_id":"semeval2013.d003.s002.t000","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":2,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":3,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":4,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":5,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":6,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"},{"index":7,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":8,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":9,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d003.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":397,"reasoning_output_tokens":382},"cost":{"currency":"USD","total_usd":0.015155,"input_uncached_usd":0.003245,"input_cached_usd":0.0,"output_usd":0.01191,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.929883207892999}
{"item_id":"semeval2013.d003.s002.t002","gold_sense_keys":["china%1:15:00::"],"candidates":[{"index":1,"sense_key":"china%1:06:00::","synset_id":"china.n.02"},{"index":2,"sense_key":"china%1:06:01::","synset_id":"chinaware.n.01"},{"index":3,"sense_key":"china%1:15:01::","synset_id":"taiwan.n.01"},{"index":4,"sense_key":"china%1:15:00::","synset_id":"china.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"china%1:15:00::","call_ids":["semeval2013.d003.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"china%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.004385,"input_uncached_usd":0.002135,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9176551250275224}
{"item_id":"semeval2013.d003.s002.t003","gold_sense_keys":["russia%1:15:04::"],"candidates":[{"index":1,"sense_key":"russia%1:15:04::","synset_id":"russia.n.04"},{"index":2,"sense_key":"russia%1:15:01::","synset_id":"soviet_russia.n.01"},{"index":3,"sense_key":"russia%1:15:00::","synset_id":"soviet_union.n.01"},{"index":4,"sense_key":"russia%1:15:02::","synset_id":"russia.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"russia%1:15:04::","call_ids":["semeval2013.d003.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"russia%1:15:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00422,"input_uncached_usd":0.0024500000000000004,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9503878330579028}
{"item_id":"semeval2013.d003.s002.t004","gold_sense_keys":["oil%1:27:01::"],"candidates":[{"index":1,"sense_key":"oil%1:27:01::","synset_id":"petroleum.n.01"},{"index":2,"sense_key":"oil%1:27:00::","synset_id":"oil.n.01"},{"index":3,"sense_key":"oil%1:06:01::","synset_id":"oil.n.02"},{"index":4,"sense_key":"oil%1:13:00::","synset_id":"vegetable_oil.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"oil%1:27:01::","call_ids":["semeval2013.d003.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"oil%1:27:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":397,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":90},"cost":{"currency":"USD","total_usd":0.005135000000000001,"input_uncached_usd":0.0019850000000000002,"input_cached_usd":0.0,"output_usd":0.00315,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4328427499858662}
{"item_id":"semeval2013.d003.s002.t006","gold_sense_keys":["role%1:04:00::"],"candidates":[{"index":1,"sense_key":"role%1:07:00::","synset_id":"function.n.02"},{"index":2,"sense_key":"role%1:04:01::","synset_id":"role.n.04"},{"index":3,"sense_key":"role%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"role%1:09:00::","synset_id":"character.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"role%1:04:00::","call_ids":["semeval2013.d003.s002.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"role%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.00439,"input_uncached_usd":0.00223,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7692875420907512}
{"item_id":"semeval2013.d003.s002.t009","gold_sense_keys":["decade%1:28:00::"],"candidates":[{"index":1,"sense_key":"decade%1:28:00::","synset_id":"decade.n.01"},{"index":2,"sense_key":"decade%1:23:00::","synset_id":"ten.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decade%1:28:00::","call_ids":["semeval2013.d003.s002.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decade%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":335,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.003475,"input_uncached_usd":0.001675,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7462205830961466}
{"item_id":"semeval2013.d003.s002.t010","gold_sense_keys":["war%1:04:00::","war%1:26:00::"],"candidates":[{"index":1,"sense_key":"war%1:26:00::","synset_id":"war.n.02"},{"index":2,"sense_key":"war%1:04:01::","synset_id":"war.n.04"},{"index":3,"sense_key":"war%1:04:02::","synset_id":"war.n.03"},{"index":4,"sense_key":"war%1:04:00::","synset_id":"war.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"war%1:04:00::","call_ids":["semeval2013.d003.s002.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"war%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.004345,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5644000839674845}
{"item_id":"semeval2013.d003.s002.t011","gold_sense_keys":["sanction%1:04:02::"],"candidates":[{"index":1,"sense_key":"sanction%1:10:00::","synset_id":"sanction.n.01"},{"index":2,"sense_key":"sanction%1:04:01::","synset_id":"sanction.n.04"},{"index":3,"sense_key":"sanction%1:04:02::","synset_id":"sanction.n.02"},{"index":4,"sense_key":"sanction%1:07:00::","synset_id":"authority.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"sanction%1:04:02::","call_ids":["semeval2013.d003.s002.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"sanction%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.004240000000000001,"input_uncached_usd":0.00202,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.013731458107941}
{"item_id":"semeval2013.d003.s003.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:14:05::","synset_id":"company.n.09"},{"index":4,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":5,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":6,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":7,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":8,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":9,"sense_key":"company%1:14:02::","synset_id":"company.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.0046700000000000005,"input_uncached_usd":0.0031100000000000004,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7338440830353647}
{"item_id":"semeval2013.d003.s003.t001","gold_sense_keys":["stake%1:21:02::"],"candidates":[{"index":1,"sense_key":"stake%1:10:00::","synset_id":"post.n.09"},{"index":2,"sense_key":"stake%1:06:01::","synset_id":"stake.n.03"},{"index":3,"sense_key":"stake%1:06:00::","synset_id":"stake.n.05"},{"index":4,"sense_key":"stake%1:21:02::","synset_id":"interest.n.05"},{"index":5,"sense_key":"stake%1:21:00::","synset_id":"stake.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"stake%1:21:02::","call_ids":["semeval2013.d003.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"stake%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.00461,"input_uncached_usd":0.0024200000000000003,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.115494208992459}
{"item_id":"semeval2013.d003.s003.t002","gold_sense_keys":["field%1:15:05::"],"candidates":[{"index":1,"sense_key":"field%1:17:00::","synset_id":"plain.n.01"},{"index":2,"sense_key":"field%1:09:00::","synset_id":"discipline.n.01"},{"index":3,"sense_key":"field%1:09:01::","synset_id":"field.n.16"},{"index":4,"sense_key":"field%1:14:01::","synset_id":"field.n.12"},{"index":5,"sense_key":"field%1:15:00::","synset_id":"field.n.01"},{"index":6,"sense_key":"field%1:06:00::","synset_id":"airfield.n.01"},{"index":7,"sense_key":"field%1:19:00::","synset_id":"field.n.05"},{"index":8,"sense_key":"field%1:04:00::","synset_id":"field.n.06"},{"index":9,"sense_key":"field%1:14:02::","synset_id":"field.n.13"},{"index":10,"sense_key":"field%1:14:00::","synset_id":"field.n.10"},{"index":11,"sense_key":"field%1:15:05::","synset_id":"field.n.14"},{"index":12,"sense_key":"field%1:15:04::","synset_id":"battlefield.n.01"},{"index":13,"sense_key":"field%1:26:00::","synset_id":"sphere.n.01"},{"index":14,"sense_key":"field%1:15:01::","synset_id":"field.n.03"},{"index":15,"sense_key":"field%1:15:03::","synset_id":"field.n.11"},{"index":16,"sense_key":"field%1:14:03::","synset_id":"field.n.15"},{"index":17,"sense_key":"field%1:15:02::","synset_id":"playing_field.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"field%1:15:05::","call_ids":["semeval2013.d003.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"field%1:15:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1043,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.007075,"input_uncached_usd":0.005215,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7420749159064144}
{"item_id":"semeval2013.d003.s004.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:05::","synset_id":"company.n.09"},{"index":2,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":5,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":6,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":7,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":8,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":9,"sense_key":"company%1:14:02::","synset_id":"company.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.004655,"input_uncached_usd":0.0031550000000000003,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3899364999961108}
{"item_id":"semeval2013.d003.s004.t001","gold_sense_keys":["auction%1:04:00::"],"candidates":[{"index":1,"sense_key":"auction%1:04:01::","synset_id":"auction.n.01"},{"index":2,"sense_key":"auction%1:04:00::","synset_id":"auction.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"auction%1:04:00::","call_ids":["semeval2013.d003.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"auction%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":358,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00362,"input_uncached_usd":0.0017900000000000001,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9425962909590453}
{"item_id":"semeval2013.d003.s005.t000","gold_sense_keys":["bid%1:10:00::"],"candidates":[{"index":1,"sense_key":"bid%1:10:03::","synset_id":"command.n.01"},{"index":2,"sense_key":"bid%1:10:00::","synset_id":"bid.n.03"},{"index":3,"sense_key":"bid%1:04:00::","synset_id":"bid.n.02"},{"index":4,"sense_key":"bid%1:10:01::","synset_id":"bid.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bid%1:10:00::","call_ids":["semeval2013.d003.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bid%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00409,"input_uncached_usd":0.00217,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5533037920249626}
{"item_id":"semeval2013.d003.s006.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":3,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":4,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":5,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":6,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":7,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":8,"sense_key":"company%1:14:05::","synset_id":"company.n.09"},{"index":9,"sense_key":"company%1:14:06::","synset_id":"party.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.005065,"input_uncached_usd":0.0032350000000000005,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.535002041957341}
{"item_id":"semeval2013.d003.s006.t001","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":3,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":4,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"},{"index":8,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":9,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d003.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":690,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":139},"cost":{"currency":"USD","total_usd":0.00807,"input_uncached_usd":0.0034500000000000004,"input_cached_usd":0.0,"output_usd":0.00462,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.177497416967526}
{"item_id":"semeval2013.d003.s006.t002","gold_sense_keys":["field%1:15:05::"],"candidates":[{"index":1,"sense_key":"field%1:15:02::","synset_id":"playing_field.n.02"},{"index":2,"sense_key":"field%1:14:00::","synset_id":"field.n.10"},{"index":3,"sense_key":"field%1:14:03::","synset_id":"field.n.15"},{"index":4,"sense_key":"field%1:09:00::","synset_id":"discipline.n.01"},{"index":5,"sense_key":"field%1:06:00::","synset_id":"airfield.n.01"},{"index":6,"sense_key":"field%1:26:00::","synset_id":"sphere.n.01"},{"index":7,"sense_key":"field%1:15:05::","synset_id":"field.n.14"},{"index":8,"sense_key":"field%1:19:00::","synset_id":"field.n.05"},{"index":9,"sense_key":"field%1:15:01::","synset_id":"field.n.03"},{"index":10,"sense_key":"field%1:14:02::","synset_id":"field.n.13"},{"index":11,"sense_key":"field%1:15:04::","synset_id":"battlefield.n.01"},{"index":12,"sense_key":"field%1:14:01::","synset_id":"field.n.12"},{"index":13,"sense_key":"field%1:17:00::","synset_id":"plain.n.01"},{"index":14,"sense_key":"field%1:15:00::","synset_id":"field.n.01"},{"index":15,"sense_key":"field%1:04:00::","synset_id":"field.n.06"},{"index":16,"sense_key":"field%1:15:03::","synset_id":"field.n.11"},{"index":17,"sense_key":"field%1:09:01::","synset_id":"field.n.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"field%1:15:05::","call_ids":["semeval2013.d003.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"field%1:15:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1069,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.0075650000000000005,"input_uncached_usd":0.005345000000000001,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9532337090931833}
{"item_id":"semeval2013.d003.s007.t000","gold_sense_keys":["representation%1:04:01::"],"candidates":[{"index":1,"sense_key":"representation%1:10:00::","synset_id":"representation.n.06"},{"index":2,"sense_key":"representation%1:10:01::","synset_id":"theatrical_performance.n.01"},{"index":3,"sense_key":"representation%1:04:00::","synset_id":"representation.n.10"},{"index":4,"sense_key":"representation%1:07:00::","synset_id":"representation.n.09"},{"index":5,"sense_key":"representation%1:26:00::","synset_id":"representation.n.04"},{"index":6,"sense_key":"representation%1:04:01::","synset_id":"representation.n.03"},{"index":7,"sense_key":"representation%1:14:00::","synset_id":"representation.n.05"},{"index":8,"sense_key":"representation%1:10:02::","synset_id":"representation.n.08"},{"index":9,"sense_key":"representation%1:09:00::","synset_id":"representation.n.01"},{"index":10,"sense_key":"representation%1:06:00::","synset_id":"representation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"representation%1:26:00::","call_ids":["semeval2013.d003.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"representation%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01918,"input_uncached_usd":0.00334,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.92377629200928}
{"item_id":"semeval2013.d003.s007.t001","gold_sense_keys":["oil%1:27:01::"],"candidates":[{"index":1,"sense_key":"oil%1:27:01::","synset_id":"petroleum.n.01"},{"index":2,"sense_key":"oil%1:13:00::","synset_id":"vegetable_oil.n.01"},{"index":3,"sense_key":"oil%1:06:01::","synset_id":"oil.n.02"},{"index":4,"sense_key":"oil%1:27:00::","synset_id":"oil.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"oil%1:27:01::","call_ids":["semeval2013.d003.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"oil%1:27:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":121},"cost":{"currency":"USD","total_usd":0.006350000000000001,"input_uncached_usd":0.0022700000000000003,"input_cached_usd":0.0,"output_usd":0.00408,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.703797208960168}
{"item_id":"semeval2013.d003.s007.t002","gold_sense_keys":["giant%1:14:00::"],"candidates":[{"index":1,"sense_key":"giant%1:18:03::","synset_id":"giant.n.04"},{"index":2,"sense_key":"giant%1:18:02::","synset_id":"giant.n.06"},{"index":3,"sense_key":"giant%1:14:00::","synset_id":"giant.n.03"},{"index":4,"sense_key":"giant%1:17:00::","synset_id":"giant_star.n.01"},{"index":5,"sense_key":"giant%1:18:01::","synset_id":"colossus.n.02"},{"index":6,"sense_key":"giant%1:18:00::","synset_id":"giant.n.05"},{"index":7,"sense_key":"giant%1:05:00::","synset_id":"giant.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"giant%1:14:00::","call_ids":["semeval2013.d003.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"giant%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.004425,"input_uncached_usd":0.002775,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":77.92392641690094}
{"item_id":"semeval2013.d003.s007.t005","gold_sense_keys":["analyst%1:18:00::","analyst%1:18:02::"],"candidates":[{"index":1,"sense_key":"analyst%1:18:01::","synset_id":"analyst.n.03"},{"index":2,"sense_key":"analyst%1:18:00::","synset_id":"analyst.n.01"},{"index":3,"sense_key":"analyst%1:18:02::","synset_id":"analyst.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"analyst%1:18:00::","call_ids":["semeval2013.d003.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"analyst%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017875000000000002,"input_uncached_usd":0.0020350000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.879935708013363}
{"item_id":"semeval2013.d003.s008.t001","gold_sense_keys":["door%1:07:00::"],"candidates":[{"index":1,"sense_key":"door%1:06:04::","synset_id":"door.n.04"},{"index":2,"sense_key":"door%1:06:00::","synset_id":"door.n.01"},{"index":3,"sense_key":"door%1:07:00::","synset_id":"door.n.03"},{"index":4,"sense_key":"door%1:06:03::","synset_id":"door.n.05"},{"index":5,"sense_key":"door%1:06:01::","synset_id":"doorway.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"door%1:07:00::","call_ids":["semeval2013.d003.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"door%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.00485,"input_uncached_usd":0.0026300000000000004,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.885543791926466}
{"item_id":"semeval2013.d003.s008.t002","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d003.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.004185,"input_uncached_usd":0.0023850000000000004,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.661708291969262}
{"item_id":"semeval2013.d003.s008.t003","gold_sense_keys":["war%1:04:00::","war%1:26:00::"],"candidates":[{"index":1,"sense_key":"war%1:26:00::","synset_id":"war.n.02"},{"index":2,"sense_key":"war%1:04:01::","synset_id":"war.n.04"},{"index":3,"sense_key":"war%1:04:02::","synset_id":"war.n.03"},{"index":4,"sense_key":"war%1:04:00::","synset_id":"war.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"war%1:04:00::","call_ids":["semeval2013.d003.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"war%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":463,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.004685,"input_uncached_usd":0.0023150000000000002,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9030273329699412}
{"item_id":"semeval2013.d003.s008.t004","gold_sense_keys":["u.s.%1:15:00::"],"candidates":[{"index":1,"sense_key":"u.s.%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"us%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"us%1:15:00::","call_ids":["semeval2013.d003.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"us%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":90},"cost":{"currency":"USD","total_usd":0.005180000000000001,"input_uncached_usd":0.002,"input_cached_usd":0.0,"output_usd":0.00318,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5214567909715697}
{"item_id":"semeval2013.d003.s008.t005","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":2,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":3,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":4,"sense_key":"company%1:14:05::","synset_id":"company.n.09"},{"index":5,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":8,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":9,"sense_key":"company%1:26:00::","synset_id":"company.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":632,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.0048400000000000006,"input_uncached_usd":0.00316,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0418475409969687}
{"item_id":"semeval2013.d003.s008.t007","gold_sense_keys":["editor%1:18:00::"],"candidates":[{"index":1,"sense_key":"editor%1:10:00::","synset_id":"editor_program.n.01"},{"index":2,"sense_key":"editor%1:18:00::","synset_id":"editor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"editor%1:18:00::","call_ids":["semeval2013.d003.s008.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"editor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":373,"cached_input_tokens":0,"output_tokens":39,"reasoning_output_tokens":24},"cost":{"currency":"USD","total_usd":0.0030350000000000004,"input_uncached_usd":0.0018650000000000001,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5467704159673303}
{"item_id":"semeval2013.d003.s008.t008","gold_sense_keys":["news%1:10:00::","news%1:10:01::"],"candidates":[{"index":1,"sense_key":"news%1:10:02::","synset_id":"news_program.n.01"},{"index":2,"sense_key":"news%1:07:00::","synset_id":"newsworthiness.n.01"},{"index":3,"sense_key":"news%1:10:03::","synset_id":"news.n.04"},{"index":4,"sense_key":"news%1:10:00::","synset_id":"news.n.01"},{"index":5,"sense_key":"news%1:10:01::","synset_id":"news.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"news%1:10:01::","call_ids":["semeval2013.d003.s008.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"news%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018349999999999998,"input_uncached_usd":0.00251,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.377877250080928}
{"item_id":"semeval2013.d003.s008.t009","gold_sense_keys":["outlet%1:06:01::"],"candidates":[{"index":1,"sense_key":"outlet%1:06:01::","synset_id":"mercantile_establishment.n.01"},{"index":2,"sense_key":"outlet%1:06:00::","synset_id":"exit.n.01"},{"index":3,"sense_key":"outlet%1:06:02::","synset_id":"wall_socket.n.01"},{"index":4,"sense_key":"outlet%1:04:00::","synset_id":"release.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"outlet%1:06:01::","call_ids":["semeval2013.d003.s008.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"outlet%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018175,"input_uncached_usd":0.0023650000000000003,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.30042158300057}
{"item_id":"semeval2013.d003.s010.t000","gold_sense_keys":["concern%1:12:01::"],"candidates":[{"index":1,"sense_key":"concern%1:09:01::","synset_id":"concern.n.04"},{"index":2,"sense_key":"concern%1:12:01::","synset_id":"concern.n.02"},{"index":3,"sense_key":"concern%1:14:00::","synset_id":"business.n.01"},{"index":4,"sense_key":"concern%1:12:00::","synset_id":"concern.n.03"},{"index":5,"sense_key":"concern%1:09:00::","synset_id":"concern.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"concern%1:12:01::","call_ids":["semeval2013.d003.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"concern%1:12:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":581,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018745,"input_uncached_usd":0.002905,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.525514082983136}
{"item_id":"semeval2013.d003.s010.t001","gold_sense_keys":["security%1:04:00::","security%1:26:00::"],"candidates":[{"index":1,"sense_key":"security%1:06:00::","synset_id":"security_system.n.02"},{"index":2,"sense_key":"security%1:21:00::","synset_id":"security.n.05"},{"index":3,"sense_key":"security%1:21:01::","synset_id":"security.n.02"},{"index":4,"sense_key":"security%1:26:00::","synset_id":"security.n.01"},{"index":5,"sense_key":"security%1:04:00::","synset_id":"security.n.09"},{"index":6,"sense_key":"security%1:14:00::","synset_id":"security.n.06"},{"index":7,"sense_key":"security%1:21:04::","synset_id":"security.n.04"},{"index":8,"sense_key":"security%1:12:00::","synset_id":"security.n.03"},{"index":9,"sense_key":"security%1:10:00::","synset_id":"security.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"security%1:26:00::","call_ids":["semeval2013.d003.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"security%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.0068850000000000005,"input_uncached_usd":0.0036750000000000003,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":115.37437333294656}
{"item_id":"semeval2013.d003.s010.t002","gold_sense_keys":["bombing%1:04:01::"],"candidates":[{"index":1,"sense_key":"bombing%1:04:00::","synset_id":"bombing.n.01"},{"index":2,"sense_key":"bombing%1:04:01::","synset_id":"bombing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bombing%1:04:01::","call_ids":["semeval2013.d003.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bombing%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":406,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.004070000000000001,"input_uncached_usd":0.00203,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6047122499439865}
{"item_id":"semeval2013.d003.s010.t004","gold_sense_keys":["instability%1:07:00::","instability%1:07:01::","instability%1:26:00::"],"candidates":[{"index":1,"sense_key":"instability%1:07:00::","synset_id":"instability.n.04"},{"index":2,"sense_key":"instability%1:26:01::","synset_id":"imbalance.n.01"},{"index":3,"sense_key":"instability%1:26:00::","synset_id":"instability.n.01"},{"index":4,"sense_key":"instability%1:07:01::","synset_id":"instability.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"instability%1:26:00::","call_ids":["semeval2013.d003.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"instability%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":468,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":115},"cost":{"currency":"USD","total_usd":0.00624,"input_uncached_usd":0.00234,"input_cached_usd":0.0,"output_usd":0.0039000000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3489613339770585}
{"item_id":"semeval2013.d003.s010.t005","gold_sense_keys":["u.s.%1:14:00::"],"candidates":[{"index":1,"sense_key":"u.s.%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"us%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"us%1:15:00::","call_ids":["semeval2013.d003.s010.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"us%1:15:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":437,"reasoning_output_tokens":421},"cost":{"currency":"USD","total_usd":0.01543,"input_uncached_usd":0.00232,"input_cached_usd":0.0,"output_usd":0.01311,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.47534491692204}
{"item_id":"semeval2013.d003.s010.t009","gold_sense_keys":["world%1:17:00::"],"candidates":[{"index":1,"sense_key":"world%1:14:01::","synset_id":"world.n.02"},{"index":2,"sense_key":"world%1:05:00::","synset_id":"world.n.08"},{"index":3,"sense_key":"world%1:17:02::","synset_id":"world.n.06"},{"index":4,"sense_key":"world%1:09:01::","synset_id":"worldly_concern.n.01"},{"index":5,"sense_key":"world%1:17:01::","synset_id":"universe.n.01"},{"index":6,"sense_key":"world%1:09:00::","synset_id":"world.n.03"},{"index":7,"sense_key":"world%1:17:00::","synset_id":"earth.n.01"},{"index":8,"sense_key":"world%1:14:02::","synset_id":"populace.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"world%1:17:00::","call_ids":["semeval2013.d003.s010.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"world%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":742,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":109},"cost":{"currency":"USD","total_usd":0.00746,"input_uncached_usd":0.00371,"input_cached_usd":0.0,"output_usd":0.0037500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9202616249676794}
{"item_id":"semeval2013.d003.s010.t011","gold_sense_keys":["analyst%1:18:00::","analyst%1:18:02::"],"candidates":[{"index":1,"sense_key":"analyst%1:18:02::","synset_id":"analyst.n.02"},{"index":2,"sense_key":"analyst%1:18:00::","synset_id":"analyst.n.01"},{"index":3,"sense_key":"analyst%1:18:01::","synset_id":"analyst.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"analyst%1:18:02::","call_ids":["semeval2013.d003.s010.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"analyst%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.033589999999999995,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":20.101677874918096}
{"item_id":"semeval2013.d003.s011.t000","gold_sense_keys":["u.s.%1:15:00::"],"candidates":[{"index":1,"sense_key":"us%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"u.s.%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"us%1:15:00::","call_ids":["semeval2013.d003.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"us%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":79},"cost":{"currency":"USD","total_usd":0.0052250000000000005,"input_uncached_usd":0.0023750000000000004,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.841644083033316}
{"item_id":"semeval2013.d003.s011.t002","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":2,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":3,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":4,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":5,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":6,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":7,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":10,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":11,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":12,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":13,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":14,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":15,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":16,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":17,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":18,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":19,"sense_key":"case%1:06:01::","synset_id":"case.n.20"},{"index":20,"sense_key":"case%1:11:00::","synset_id":"case.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":20,"chosen_sense_key":"case%1:11:00::","call_ids":["semeval2013.d003.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":20,"predicted_sense_key":"case%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1218,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.008490000000000001,"input_uncached_usd":0.006090000000000001,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.169186459039338}
{"item_id":"semeval2013.d003.s011.t005","gold_sense_keys":["government%1:14:00::"],"candidates":[{"index":1,"sense_key":"government%1:09:01::","synset_id":"politics.n.02"},{"index":2,"sense_key":"government%1:09:00::","synset_id":"government.n.03"},{"index":3,"sense_key":"government%1:04:00::","synset_id":"government.n.02"},{"index":4,"sense_key":"government%1:14:00::","synset_id":"government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"government%1:14:00::","call_ids":["semeval2013.d003.s011.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"government%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.005085000000000001,"input_uncached_usd":0.0026850000000000003,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.981997916009277}
{"item_id":"semeval2013.d003.s011.t006","gold_sense_keys":["energy%1:19:01::"],"candidates":[{"index":1,"sense_key":"energy%1:14:00::","synset_id":"department_of_energy.n.01"},{"index":2,"sense_key":"energy%1:26:00::","synset_id":"energy.n.05"},{"index":3,"sense_key":"energy%1:07:00::","synset_id":"energy.n.02"},{"index":4,"sense_key":"energy%1:19:00::","synset_id":"energy.n.01"},{"index":5,"sense_key":"energy%1:07:02::","synset_id":"energy.n.03"},{"index":6,"sense_key":"energy%1:19:01::","synset_id":"energy.n.06"},{"index":7,"sense_key":"energy%1:07:01::","synset_id":"energy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"energy%1:19:01::","call_ids":["semeval2013.d003.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"energy%1:19:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":112},"cost":{"currency":"USD","total_usd":0.007265000000000001,"input_uncached_usd":0.003425,"input_cached_usd":0.0,"output_usd":0.00384,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.232733374927193}
{"item_id":"semeval2013.d003.s012.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":2,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":3,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":6,"sense_key":"company%1:14:05::","synset_id":"company.n.09"},{"index":7,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":8,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":9,"sense_key":"company%1:14:03::","synset_id":"company.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":734,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.00517,"input_uncached_usd":0.00367,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.457449249923229}
{"item_id":"semeval2013.d003.s012.t001","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":2,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":3,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":4,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":5,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":6,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":7,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"},{"index":8,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":9,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d003.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":777,"cached_input_tokens":0,"output_tokens":164,"reasoning_output_tokens":148},"cost":{"currency":"USD","total_usd":0.008805,"input_uncached_usd":0.0038850000000000004,"input_cached_usd":0.0,"output_usd":0.00492,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2389134580735117}
{"item_id":"semeval2013.d003.s012.t002","gold_sense_keys":["ministry%1:14:00::"],"candidates":[{"index":1,"sense_key":"ministry%1:14:00::","synset_id":"ministry.n.03"},{"index":2,"sense_key":"ministry%1:04:00::","synset_id":"ministry.n.04"},{"index":3,"sense_key":"ministry%1:14:01::","synset_id":"ministry.n.01"},{"index":4,"sense_key":"ministry%1:06:00::","synset_id":"ministry.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ministry%1:14:00::","call_ids":["semeval2013.d003.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ministry%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.00468,"input_uncached_usd":0.0024600000000000004,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4455740000121295}
{"item_id":"semeval2013.d003.s013.t000","gold_sense_keys":["u.s.%1:15:00::"],"candidates":[{"index":1,"sense_key":"u.s.%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"us%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"us%1:15:00::","call_ids":["semeval2013.d003.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"us%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":95},"cost":{"currency":"USD","total_usd":0.00583,"input_uncached_usd":0.00253,"input_cached_usd":0.0,"output_usd":0.0033,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.42672487499658}
{"item_id":"semeval2013.d003.s013.t002","gold_sense_keys":["tie%1:26:01::"],"candidates":[{"index":1,"sense_key":"tie%1:11:00::","synset_id":"draw.n.03"},{"index":2,"sense_key":"tie%1:10:00::","synset_id":"tie.n.07"},{"index":3,"sense_key":"tie%1:06:04::","synset_id":"link.n.02"},{"index":4,"sense_key":"tie%1:06:03::","synset_id":"tie.n.08"},{"index":5,"sense_key":"tie%1:06:02::","synset_id":"tie.n.04"},{"index":6,"sense_key":"tie%1:26:00::","synset_id":"tie.n.03"},{"index":7,"sense_key":"tie%1:26:01::","synset_id":"affiliation.n.01"},{"index":8,"sense_key":"tie%1:06:00::","synset_id":"tie.n.09"},{"index":9,"sense_key":"tie%1:06:01::","synset_id":"necktie.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"tie%1:26:01::","call_ids":["semeval2013.d003.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"tie%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":838,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.00578,"input_uncached_usd":0.00419,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.226941999979317}
{"item_id":"semeval2013.d003.s013.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":2,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d003.s013.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.0051,"input_uncached_usd":0.0029100000000000003,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6772727500647306}
{"item_id":"semeval2013.d003.s014.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":2,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":3,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":4,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":5,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":6,"sense_key":"company%1:14:05::","synset_id":"company.n.09"},{"index":7,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":8,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":9,"sense_key":"company%1:26:00::","synset_id":"company.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":719,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.005275,"input_uncached_usd":0.003595,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7464372089598328}
{"item_id":"semeval2013.d003.s014.t001","gold_sense_keys":["contract%1:10:00::"],"candidates":[{"index":1,"sense_key":"contract%1:10:01::","synset_id":"contract.n.02"},{"index":2,"sense_key":"contract%1:04:01::","synset_id":"contract.n.03"},{"index":3,"sense_key":"contract%1:10:00::","synset_id":"contract.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"contract%1:10:00::","call_ids":["semeval2013.d003.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"contract%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":41,"reasoning_output_tokens":25},"cost":{"currency":"USD","total_usd":0.0036249999999999998,"input_uncached_usd":0.002395,"input_cached_usd":0.0,"output_usd":0.00123,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4932787499856204}
{"item_id":"semeval2013.d003.s015.t000","gold_sense_keys":["bid%2:40:00::"],"candidates":[{"index":1,"sense_key":"bid%2:32:00::","synset_id":"invite.v.04"},{"index":2,"sense_key":"bid%2:32:01::","synset_id":"wish.v.06"},{"index":3,"sense_key":"bid%2:40:04::","synset_id":"bid.v.05"},{"index":4,"sense_key":"bid%2:32:02::","synset_id":"bid.v.03"},{"index":5,"sense_key":"bid%2:40:01::","synset_id":"bid.v.04"},{"index":6,"sense_key":"bid%2:40:00::","synset_id":"offer.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"bid%2:40:00::","call_ids":["semeval2013.d003.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"bid%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":119},"cost":{"currency":"USD","total_usd":0.007120000000000001,"input_uncached_usd":0.0030700000000000002,"input_cached_usd":0.0,"output_usd":0.00405,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.602129708044231}
{"item_id":"semeval2013.d003.s015.t001","gold_sense_keys":["contract%1:10:00::"],"candidates":[{"index":1,"sense_key":"contract%1:10:01::","synset_id":"contract.n.02"},{"index":2,"sense_key":"contract%1:04:01::","synset_id":"contract.n.03"},{"index":3,"sense_key":"contract%1:10:00::","synset_id":"contract.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"contract%1:10:00::","call_ids":["semeval2013.d003.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"contract%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":42,"reasoning_output_tokens":27},"cost":{"currency":"USD","total_usd":0.0037100000000000006,"input_uncached_usd":0.0024500000000000004,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3878254579612985}
{"item_id":"semeval2013.d003.s015.t002","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":2,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":3,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":4,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":5,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":6,"sense_key":"company%1:14:05::","synset_id":"company.n.09"},{"index":7,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":8,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":9,"sense_key":"company%1:14:00::","synset_id":"company.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.0053300000000000005,"input_uncached_usd":0.0036500000000000005,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5084239999996498}
{"item_id":"semeval2013.d003.s015.t003","gold_sense_keys":["stake%1:21:02::"],"candidates":[{"index":1,"sense_key":"stake%1:06:00::","synset_id":"stake.n.05"},{"index":2,"sense_key":"stake%1:06:01::","synset_id":"stake.n.03"},{"index":3,"sense_key":"stake%1:10:00::","synset_id":"post.n.09"},{"index":4,"sense_key":"stake%1:21:02::","synset_id":"interest.n.05"},{"index":5,"sense_key":"stake%1:21:00::","synset_id":"stake.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"stake%1:21:02::","call_ids":["semeval2013.d003.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"stake%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.0049700000000000005,"input_uncached_usd":0.0029600000000000004,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.077260790974833}
{"item_id":"semeval2013.d003.s015.t004","gold_sense_keys":["contract%1:10:00::"],"candidates":[{"index":1,"sense_key":"contract%1:10:01::","synset_id":"contract.n.02"},{"index":2,"sense_key":"contract%1:04:01::","synset_id":"contract.n.03"},{"index":3,"sense_key":"contract%1:10:00::","synset_id":"contract.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"contract%1:10:00::","call_ids":["semeval2013.d003.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"contract%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":36,"reasoning_output_tokens":20},"cost":{"currency":"USD","total_usd":0.00353,"input_uncached_usd":0.0024500000000000004,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.1212926250882447}
{"item_id":"semeval2013.d003.s015.t005","gold_sense_keys":["field%1:15:05::"],"candidates":[{"index":1,"sense_key":"field%1:14:00::","synset_id":"field.n.10"},{"index":2,"sense_key":"field%1:15:04::","synset_id":"battlefield.n.01"},{"index":3,"sense_key":"field%1:19:00::","synset_id":"field.n.05"},{"index":4,"sense_key":"field%1:15:05::","synset_id":"field.n.14"},{"index":5,"sense_key":"field%1:09:00::","synset_id":"discipline.n.01"},{"index":6,"sense_key":"field%1:14:02::","synset_id":"field.n.13"},{"index":7,"sense_key":"field%1:15:03::","synset_id":"field.n.11"},{"index":8,"sense_key":"field%1:15:01::","synset_id":"field.n.03"},{"index":9,"sense_key":"field%1:06:00::","synset_id":"airfield.n.01"},{"index":10,"sense_key":"field%1:15:00::","synset_id":"field.n.01"},{"index":11,"sense_key":"field%1:26:00::","synset_id":"sphere.n.01"},{"index":12,"sense_key":"field%1:04:00::","synset_id":"field.n.06"},{"index":13,"sense_key":"field%1:14:01::","synset_id":"field.n.12"},{"index":14,"sense_key":"field%1:14:03::","synset_id":"field.n.15"},{"index":15,"sense_key":"field%1:17:00::","synset_id":"plain.n.01"},{"index":16,"sense_key":"field%1:15:02::","synset_id":"playing_field.n.02"},{"index":17,"sense_key":"field%1:09:01::","synset_id":"field.n.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"field%1:15:05::","call_ids":["semeval2013.d003.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"field%1:15:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1151,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.007765000000000001,"input_uncached_usd":0.005755000000000001,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1153215839294717}
{"item_id":"semeval2013.d003.s016.t000","gold_sense_keys":["china%1:15:00::"],"candidates":[{"index":1,"sense_key":"china%1:06:01::","synset_id":"chinaware.n.01"},{"index":2,"sense_key":"china%1:06:00::","synset_id":"china.n.02"},{"index":3,"sense_key":"china%1:15:00::","synset_id":"china.n.01"},{"index":4,"sense_key":"china%1:15:01::","synset_id":"taiwan.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"china%1:15:00::","call_ids":["semeval2013.d003.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"china%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":39,"reasoning_output_tokens":23},"cost":{"currency":"USD","total_usd":0.003775,"input_uncached_usd":0.002605,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.543260082951747}
{"item_id":"semeval2013.d003.s016.t002","gold_sense_keys":["power%1:14:00::"],"candidates":[{"index":1,"sense_key":"power%1:14:00::","synset_id":"world_power.n.01"},{"index":2,"sense_key":"power%1:18:00::","synset_id":"power.n.05"},{"index":3,"sense_key":"power%1:09:01::","synset_id":"ability.n.02"},{"index":4,"sense_key":"power%1:07:00::","synset_id":"power.n.01"},{"index":5,"sense_key":"power%1:26:00::","synset_id":"office.n.04"},{"index":6,"sense_key":"power%1:19:00::","synset_id":"power.n.02"},{"index":7,"sense_key":"power%1:18:01::","synset_id":"baron.n.03"},{"index":8,"sense_key":"power%1:07:02::","synset_id":"might.n.01"},{"index":9,"sense_key":"power%1:10:00::","synset_id":"exponent.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"power%1:14:00::","call_ids":["semeval2013.d003.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"power%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":108},"cost":{"currency":"USD","total_usd":0.007345000000000001,"input_uncached_usd":0.003625,"input_cached_usd":0.0,"output_usd":0.00372,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6086573330685496}
{"item_id":"semeval2013.d003.s017.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:05::","synset_id":"company.n.09"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":4,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":5,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":6,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":7,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":8,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":9,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":690,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.005220000000000001,"input_uncached_usd":0.0034500000000000004,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.781358625041321}
{"item_id":"semeval2013.d003.s017.t001","gold_sense_keys":["par%1:26:00::"],"candidates":[{"index":1,"sense_key":"par%1:26:00::","synset_id":"equality.n.02"},{"index":2,"sense_key":"par%1:23:00::","synset_id":"par.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"par%1:26:00::","call_ids":["semeval2013.d003.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"par%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.003625,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4028003750136122}
{"item_id":"semeval2013.d003.s017.t003","gold_sense_keys":["commitment%1:04:02::"],"candidates":[{"index":1,"sense_key":"commitment%1:04:01::","synset_id":"commitment.n.05"},{"index":2,"sense_key":"commitment%1:10:00::","synset_id":"commitment.n.04"},{"index":3,"sense_key":"commitment%1:07:01::","synset_id":"committedness.n.01"},{"index":4,"sense_key":"commitment%1:04:02::","synset_id":"commitment.n.03"},{"index":5,"sense_key":"commitment%1:04:00::","synset_id":"commitment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"commitment%1:04:02::","call_ids":["semeval2013.d003.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"commitment%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":168,"reasoning_output_tokens":152},"cost":{"currency":"USD","total_usd":0.007805,"input_uncached_usd":0.002765,"input_cached_usd":0.0,"output_usd":0.00504,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.620530875050463}
{"item_id":"semeval2013.d003.s018.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":2,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":5,"sense_key":"company%1:14:05::","synset_id":"company.n.09"},{"index":6,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":7,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":8,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":9,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.005205,"input_uncached_usd":0.0033450000000000003,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3603329170728102}
{"item_id":"semeval2013.d003.s018.t001","gold_sense_keys":["bid%1:10:00::"],"candidates":[{"index":1,"sense_key":"bid%1:10:03::","synset_id":"command.n.01"},{"index":2,"sense_key":"bid%1:10:00::","synset_id":"bid.n.03"},{"index":3,"sense_key":"bid%1:04:00::","synset_id":"bid.n.02"},{"index":4,"sense_key":"bid%1:10:01::","synset_id":"bid.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bid%1:10:00::","call_ids":["semeval2013.d003.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bid%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.004280000000000001,"input_uncached_usd":0.0023000000000000004,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6487312089884654}
{"item_id":"semeval2013.d003.s018.t002","gold_sense_keys":["offer%1:10:00::"],"candidates":[{"index":1,"sense_key":"offer%1:10:01::","synset_id":"offer.n.01"},{"index":2,"sense_key":"offer%1:04:01::","synset_id":"crack.n.09"},{"index":3,"sense_key":"offer%1:10:00::","synset_id":"offer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"offer%1:10:00::","call_ids":["semeval2013.d003.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"offer%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":101},"cost":{"currency":"USD","total_usd":0.005725000000000001,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.373806416988373}
{"item_id":"semeval2013.d003.s018.t003","gold_sense_keys":["barrel%1:23:01::"],"candidates":[{"index":1,"sense_key":"barrel%1:23:01::","synset_id":"barrel.n.05"},{"index":2,"sense_key":"barrel%1:23:04::","synset_id":"barrel.n.04"},{"index":3,"sense_key":"barrel%1:25:00::","synset_id":"barrel.n.03"},{"index":4,"sense_key":"barrel%1:06:00::","synset_id":"barrel.n.02"},{"index":5,"sense_key":"barrel%1:06:01::","synset_id":"barrel.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"barrel%1:23:01::","call_ids":["semeval2013.d003.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"barrel%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.004150000000000001,"input_uncached_usd":0.0024700000000000004,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7353350420016795}
{"item_id":"semeval2013.d003.s018.t004","gold_sense_keys":["fee%1:21:00::"],"candidates":[{"index":1,"sense_key":"fee%1:21:01::","synset_id":"fee.n.02"},{"index":2,"sense_key":"fee%1:21:00::","synset_id":"fee.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fee%1:21:00::","call_ids":["semeval2013.d003.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fee%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":368,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.00373,"input_uncached_usd":0.00184,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0034457920119166}
{"item_id":"semeval2013.d003.s018.t005","gold_sense_keys":["ministry%1:14:00::"],"candidates":[{"index":1,"sense_key":"ministry%1:14:01::","synset_id":"ministry.n.01"},{"index":2,"sense_key":"ministry%1:04:00::","synset_id":"ministry.n.04"},{"index":3,"sense_key":"ministry%1:06:00::","synset_id":"ministry.n.02"},{"index":4,"sense_key":"ministry%1:14:00::","synset_id":"ministry.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"ministry%1:14:00::","call_ids":["semeval2013.d003.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"ministry%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.0043,"input_uncached_usd":0.00214,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6238948330283165}
{"item_id":"semeval2013.d003.s018.t006","gold_sense_keys":["output%1:04:00::","output%1:06:01::","output%1:23:00::"],"candidates":[{"index":1,"sense_key":"output%1:23:00::","synset_id":"output.n.04"},{"index":2,"sense_key":"output%1:04:00::","synset_id":"output.n.02"},{"index":3,"sense_key":"output%1:06:01::","synset_id":"output.n.05"},{"index":4,"sense_key":"output%1:06:00::","synset_id":"end_product.n.01"},{"index":5,"sense_key":"output%1:10:02::","synset_id":"output_signal.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"output%1:23:00::","call_ids":["semeval2013.d003.s018.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"output%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":131},"cost":{"currency":"USD","total_usd":0.006840000000000001,"input_uncached_usd":0.0024300000000000003,"input_cached_usd":0.0,"output_usd":0.00441,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4154654169688}
{"item_id":"semeval2013.d003.s018.t007","gold_sense_keys":["level%1:07:00::"],"candidates":[{"index":1,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":2,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":3,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":4,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"level%1:07:00::","call_ids":["semeval2013.d003.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"level%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":679,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.006485,"input_uncached_usd":0.0033950000000000004,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.46344850002788}
{"item_id":"semeval2013.d003.s018.t008","gold_sense_keys":["field%1:15:05::"],"candidates":[{"index":1,"sense_key":"field%1:09:01::","synset_id":"field.n.16"},{"index":2,"sense_key":"field%1:15:01::","synset_id":"field.n.03"},{"index":3,"sense_key":"field%1:19:00::","synset_id":"field.n.05"},{"index":4,"sense_key":"field%1:14:03::","synset_id":"field.n.15"},{"index":5,"sense_key":"field%1:14:02::","synset_id":"field.n.13"},{"index":6,"sense_key":"field%1:14:01::","synset_id":"field.n.12"},{"index":7,"sense_key":"field%1:15:03::","synset_id":"field.n.11"},{"index":8,"sense_key":"field%1:15:02::","synset_id":"playing_field.n.02"},{"index":9,"sense_key":"field%1:14:00::","synset_id":"field.n.10"},{"index":10,"sense_key":"field%1:17:00::","synset_id":"plain.n.01"},{"index":11,"sense_key":"field%1:04:00::","synset_id":"field.n.06"},{"index":12,"sense_key":"field%1:15:00::","synset_id":"field.n.01"},{"index":13,"sense_key":"field%1:15:05::","synset_id":"field.n.14"},{"index":14,"sense_key":"field%1:15:04::","synset_id":"battlefield.n.01"},{"index":15,"sense_key":"field%1:06:00::","synset_id":"airfield.n.01"},{"index":16,"sense_key":"field%1:26:00::","synset_id":"sphere.n.01"},{"index":17,"sense_key":"field%1:09:00::","synset_id":"discipline.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"field%1:15:05::","call_ids":["semeval2013.d003.s018.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"field%1:15:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1091,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.007585000000000001,"input_uncached_usd":0.005455000000000001,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.876940125017427}
{"item_id":"semeval2013.d004.s000.t000","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":2,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":3,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":4,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":5,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"},{"index":6,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":7,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":8,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":9,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":10,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.00456,"input_uncached_usd":0.0030600000000000002,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4701919169165194}
{"item_id":"semeval2013.d004.s000.t001","gold_sense_keys":["state%1:14:01::"],"candidates":[{"index":1,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":2,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":3,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"},{"index":4,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":5,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":6,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":7,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":8,"sense_key":"state%1:14:00::","synset_id":"state.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"state%1:14:01::","call_ids":["semeval2013.d004.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"state%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":111},"cost":{"currency":"USD","total_usd":0.006580000000000001,"input_uncached_usd":0.0028000000000000004,"input_cached_usd":0.0,"output_usd":0.00378,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9666872089728713}
{"item_id":"semeval2013.d004.s000.t002","gold_sense_keys":["support%1:21:01::"],"candidates":[{"index":1,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":2,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:21:01::","synset_id":"support.n.11"},{"index":5,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":6,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":7,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":8,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":9,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":10,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":11,"sense_key":"support%1:04:04::","synset_id":"support.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"support%1:21:01::","call_ids":["semeval2013.d004.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"support%1:21:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":166,"reasoning_output_tokens":150},"cost":{"currency":"USD","total_usd":0.008385000000000002,"input_uncached_usd":0.0034050000000000005,"input_cached_usd":0.0,"output_usd":0.00498,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.37206750002224}
{"item_id":"semeval2013.d004.s001.t001","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":2,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":3,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":6,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":7,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"},{"index":8,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":9,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":10,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.004675,"input_uncached_usd":0.0032350000000000005,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.119818542036228}
{"item_id":"semeval2013.d004.s001.t002","gold_sense_keys":["state%1:14:00::","state%1:14:01::"],"candidates":[{"index":1,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":2,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":3,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":4,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":5,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"},{"index":6,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":7,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":8,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"state%1:14:01::","call_ids":["semeval2013.d004.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"state%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":141},"cost":{"currency":"USD","total_usd":0.007685000000000001,"input_uncached_usd":0.002975,"input_cached_usd":0.0,"output_usd":0.00471,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7750211249804124}
{"item_id":"semeval2013.d004.s001.t003","gold_sense_keys":["shackle%1:06:00::"],"candidates":[{"index":1,"sense_key":"shackle%1:06:00::","synset_id":"shackle.n.01"},{"index":2,"sense_key":"shackle%1:06:01::","synset_id":"shackle.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"shackle%1:06:00::","call_ids":["semeval2013.d004.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"shackle%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":289,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.003875,"input_uncached_usd":0.001445,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8495530420914292}
{"item_id":"semeval2013.d004.s002.t001","gold_sense_keys":["support%1:21:01::"],"candidates":[{"index":1,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":2,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":5,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":6,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":7,"sense_key":"support%1:21:01::","synset_id":"support.n.11"},{"index":8,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":9,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:06:01::","synset_id":"support.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"support%1:21:01::","call_ids":["semeval2013.d004.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"support%1:21:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":188,"reasoning_output_tokens":172},"cost":{"currency":"USD","total_usd":0.009325000000000002,"input_uncached_usd":0.0036850000000000003,"input_cached_usd":0.0,"output_usd":0.00564,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.842134458012879}
{"item_id":"semeval2013.d004.s002.t002","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":4,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":5,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":6,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":7,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":8,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":9,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":10,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":86},"cost":{"currency":"USD","total_usd":0.0064,"input_uncached_usd":0.00334,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.550047958036885}
{"item_id":"semeval2013.d004.s002.t003","gold_sense_keys":["protection%1:21:00::"],"candidates":[{"index":1,"sense_key":"protection%1:04:01::","synset_id":"protection.n.07"},{"index":2,"sense_key":"protection%1:21:00::","synset_id":"security.n.02"},{"index":3,"sense_key":"protection%1:04:03::","synset_id":"protection.n.06"},{"index":4,"sense_key":"protection%1:26:00::","synset_id":"protection.n.04"},{"index":5,"sense_key":"protection%1:06:00::","synset_id":"protective_covering.n.01"},{"index":6,"sense_key":"protection%1:04:00::","synset_id":"protection.n.01"},{"index":7,"sense_key":"protection%1:04:02::","synset_id":"auspices.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"protection%1:21:00::","call_ids":["semeval2013.d004.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"protection%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":109},"cost":{"currency":"USD","total_usd":0.006565,"input_uncached_usd":0.0028150000000000002,"input_cached_usd":0.0,"output_usd":0.0037500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1010818749200553}
{"item_id":"semeval2013.d004.s002.t004","gold_sense_keys":["fund%1:21:00::"],"candidates":[{"index":1,"sense_key":"fund%1:21:00::","synset_id":"fund.n.01"},{"index":2,"sense_key":"fund%1:14:00::","synset_id":"investment_company.n.01"},{"index":3,"sense_key":"fund%1:21:01::","synset_id":"store.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fund%1:21:00::","call_ids":["semeval2013.d004.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fund%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":342,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.0035700000000000003,"input_uncached_usd":0.0017100000000000001,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6420903750695288}
{"item_id":"semeval2013.d004.s002.t005","gold_sense_keys":["amount%1:21:00::"],"candidates":[{"index":1,"sense_key":"amount%1:03:00::","synset_id":"measure.n.02"},{"index":2,"sense_key":"amount%1:09:00::","synset_id":"sum.n.02"},{"index":3,"sense_key":"amount%1:07:00::","synset_id":"amount.n.02"},{"index":4,"sense_key":"amount%1:21:00::","synset_id":"sum.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"amount%1:21:00::","call_ids":["semeval2013.d004.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"amount%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":380,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.0037600000000000003,"input_uncached_usd":0.0019000000000000002,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5637218750780448}
{"item_id":"semeval2013.d004.s003.t000","gold_sense_keys":["report%1:10:00::"],"candidates":[{"index":1,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":2,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":3,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":4,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":5,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":6,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"},{"index":7,"sense_key":"report%1:11:00::","synset_id":"report.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"report%1:10:00::","call_ids":["semeval2013.d004.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"report%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":160,"reasoning_output_tokens":144},"cost":{"currency":"USD","total_usd":0.007730000000000001,"input_uncached_usd":0.0029300000000000003,"input_cached_usd":0.0,"output_usd":0.0048000000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0320397089235485}
{"item_id":"semeval2013.d004.s003.t002","gold_sense_keys":["broadcaster%1:18:00::"],"candidates":[{"index":1,"sense_key":"broadcaster%1:18:00::","synset_id":"broadcaster.n.01"},{"index":2,"sense_key":"broadcaster%1:06:00::","synset_id":"broadcaster.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"broadcaster%1:18:00::","call_ids":["semeval2013.d004.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"broadcaster%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":326,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.00385,"input_uncached_usd":0.0016300000000000002,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.714926583925262}
{"item_id":"semeval2013.d004.s003.t003","gold_sense_keys":["state%1:14:01::"],"candidates":[{"index":1,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"},{"index":2,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":6,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":7,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":8,"sense_key":"state%1:14:01::","synset_id":"state.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"state%1:14:01::","call_ids":["semeval2013.d004.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"state%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":155,"reasoning_output_tokens":139},"cost":{"currency":"USD","total_usd":0.007915,"input_uncached_usd":0.003265,"input_cached_usd":0.0,"output_usd":0.0046500000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.373510583071038}
{"item_id":"semeval2013.d004.s003.t004","gold_sense_keys":["support%1:21:01::"],"candidates":[{"index":1,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":2,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":3,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":4,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":5,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":6,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":9,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":10,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"support%1:21:01::","call_ids":["semeval2013.d004.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"support%1:21:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":160,"reasoning_output_tokens":144},"cost":{"currency":"USD","total_usd":0.008665,"input_uncached_usd":0.0038650000000000004,"input_cached_usd":0.0,"output_usd":0.0048000000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1762813340174034}
{"item_id":"semeval2013.d004.s004.t000","gold_sense_keys":["institution%1:14:00::"],"candidates":[{"index":1,"sense_key":"institution%1:09:00::","synset_id":"institution.n.03"},{"index":2,"sense_key":"institution%1:06:01::","synset_id":"mental_hospital.n.01"},{"index":3,"sense_key":"institution%1:04:00::","synset_id":"initiation.n.02"},{"index":4,"sense_key":"institution%1:06:00::","synset_id":"institution.n.02"},{"index":5,"sense_key":"institution%1:14:00::","synset_id":"institution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"institution%1:14:00::","call_ids":["semeval2013.d004.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"institution%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.0051199999999999996,"input_uncached_usd":0.00251,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.625858374987729}
{"item_id":"semeval2013.d004.s004.t001","gold_sense_keys":["restriction%1:09:00::"],"candidates":[{"index":1,"sense_key":"restriction%1:09:00::","synset_id":"restriction.n.01"},{"index":2,"sense_key":"restriction%1:04:00::","synset_id":"limitation.n.05"},{"index":3,"sense_key":"restriction%1:04:01::","synset_id":"restriction.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"restriction%1:09:00::","call_ids":["semeval2013.d004.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"restriction%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":168,"reasoning_output_tokens":152},"cost":{"currency":"USD","total_usd":0.007050000000000001,"input_uncached_usd":0.00201,"input_cached_usd":0.0,"output_usd":0.00504,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7026361250318587}
{"item_id":"semeval2013.d004.s004.t002","gold_sense_keys":["acceptance%1:04:03::"],"candidates":[{"index":1,"sense_key":"acceptance%1:04:03::","synset_id":"acceptance.n.07"},{"index":2,"sense_key":"acceptance%1:21:00::","synset_id":"acceptance.n.05"},{"index":3,"sense_key":"acceptance%1:26:00::","synset_id":"acceptance.n.03"},{"index":4,"sense_key":"acceptance%1:09:00::","synset_id":"credence.n.01"},{"index":5,"sense_key":"acceptance%1:04:00::","synset_id":"adoption.n.01"},{"index":6,"sense_key":"acceptance%1:10:01::","synset_id":"acceptance.n.04"},{"index":7,"sense_key":"acceptance%1:07:00::","synset_id":"toleration.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"acceptance%1:04:03::","call_ids":["semeval2013.d004.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"acceptance%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.004845,"input_uncached_usd":0.002955,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9084205830004066}
{"item_id":"semeval2013.d004.s004.t003","gold_sense_keys":["money%1:21:00::"],"candidates":[{"index":1,"sense_key":"money%1:21:01::","synset_id":"money.n.03"},{"index":2,"sense_key":"money%1:21:02::","synset_id":"money.n.02"},{"index":3,"sense_key":"money%1:21:00::","synset_id":"money.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"money%1:21:00::","call_ids":["semeval2013.d004.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"money%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":386,"cached_input_tokens":0,"output_tokens":155,"reasoning_output_tokens":139},"cost":{"currency":"USD","total_usd":0.006580000000000001,"input_uncached_usd":0.00193,"input_cached_usd":0.0,"output_usd":0.0046500000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6736236250726506}
{"item_id":"semeval2013.d004.s004.t004","gold_sense_keys":["limit%1:23:00::"],"candidates":[{"index":1,"sense_key":"limit%1:23:00::","synset_id":"limit.n.06"},{"index":2,"sense_key":"limit%1:28:00::","synset_id":"terminus_ad_quem.n.01"},{"index":3,"sense_key":"limit%1:07:00::","synset_id":"limit.n.01"},{"index":4,"sense_key":"limit%1:15:00::","synset_id":"limit.n.04"},{"index":5,"sense_key":"limit%1:15:02::","synset_id":"limit.n.03"},{"index":6,"sense_key":"limit%1:23:01::","synset_id":"limit.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"limit%1:23:00::","call_ids":["semeval2013.d004.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"limit%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004555,"input_uncached_usd":0.0025450000000000004,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8787581250071526}
{"item_id":"semeval2013.d004.s004.t005","gold_sense_keys":["manager%1:18:00::"],"candidates":[{"index":1,"sense_key":"manager%1:18:01::","synset_id":"coach.n.01"},{"index":2,"sense_key":"manager%1:18:00::","synset_id":"director.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"manager%1:18:00::","call_ids":["semeval2013.d004.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"manager%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":342,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0036300000000000004,"input_uncached_usd":0.0017100000000000001,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.035888959071599}
{"item_id":"semeval2013.d004.s004.t006","gold_sense_keys":["bonus%1:21:00::"],"candidates":[{"index":1,"sense_key":"bonus%1:09:00::","synset_id":"bonus.n.01"},{"index":2,"sense_key":"bonus%1:21:00::","synset_id":"bonus.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bonus%1:21:00::","call_ids":["semeval2013.d004.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bonus%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":354,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.00345,"input_uncached_usd":0.00177,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5322899579769}
{"item_id":"semeval2013.d004.s004.t007","gold_sense_keys":["charge%1:21:02::"],"candidates":[{"index":1,"sense_key":"charge%1:12:00::","synset_id":"bang.n.04"},{"index":2,"sense_key":"charge%1:10:01::","synset_id":"commission.n.06"},{"index":3,"sense_key":"charge%1:21:01::","synset_id":"charge.n.08"},{"index":4,"sense_key":"charge%1:21:02::","synset_id":"charge.n.03"},{"index":5,"sense_key":"charge%1:04:00::","synset_id":"care.n.05"},{"index":6,"sense_key":"charge%1:06:00::","synset_id":"charge.n.15"},{"index":7,"sense_key":"charge%1:16:00::","synset_id":"cathexis.n.01"},{"index":8,"sense_key":"charge%1:19:00::","synset_id":"charge.n.04"},{"index":9,"sense_key":"charge%1:10:00::","synset_id":"charge.n.02"},{"index":10,"sense_key":"charge%1:10:03::","synset_id":"charge.n.11"},{"index":11,"sense_key":"charge%1:18:00::","synset_id":"charge.n.07"},{"index":12,"sense_key":"charge%1:06:01::","synset_id":"charge.n.14"},{"index":13,"sense_key":"charge%1:04:01::","synset_id":"charge.n.01"},{"index":14,"sense_key":"charge%1:04:03::","synset_id":"mission.n.03"},{"index":15,"sense_key":"charge%1:10:04::","synset_id":"accusation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"charge%1:21:02::","call_ids":["semeval2013.d004.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"charge%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":949,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020585,"input_uncached_usd":0.004745,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.29452366696205}
{"item_id":"semeval2013.d004.s005.t001","gold_sense_keys":["loan%1:21:00::"],"candidates":[{"index":1,"sense_key":"loan%1:10:00::","synset_id":"loanword.n.01"},{"index":2,"sense_key":"loan%1:21:00::","synset_id":"loan.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"loan%1:21:00::","call_ids":["semeval2013.d004.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"loan%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":369,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.004035,"input_uncached_usd":0.001845,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6178998749237508}
{"item_id":"semeval2013.d004.s006.t000","gold_sense_keys":["rest%1:24:00::"],"candidates":[{"index":1,"sense_key":"rest%1:24:00::","synset_id":"remainder.n.01"},{"index":2,"sense_key":"rest%1:04:00::","synset_id":"rest.n.02"},{"index":3,"sense_key":"rest%1:28:00::","synset_id":"respite.n.04"},{"index":4,"sense_key":"rest%1:26:00::","synset_id":"rest.n.04"},{"index":5,"sense_key":"rest%1:10:00::","synset_id":"rest.n.07"},{"index":6,"sense_key":"rest%1:26:01::","synset_id":"rest.n.05"},{"index":7,"sense_key":"rest%1:06:00::","synset_id":"rest.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rest%1:24:00::","call_ids":["semeval2013.d004.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rest%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.0049250000000000006,"input_uncached_usd":0.0031550000000000003,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2975774579681456}
{"item_id":"semeval2013.d004.s006.t001","gold_sense_keys":["amount%1:21:00::"],"candidates":[{"index":1,"sense_key":"amount%1:03:00::","synset_id":"measure.n.02"},{"index":2,"sense_key":"amount%1:07:00::","synset_id":"amount.n.02"},{"index":3,"sense_key":"amount%1:09:00::","synset_id":"sum.n.02"},{"index":4,"sense_key":"amount%1:21:00::","synset_id":"sum.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"amount%1:21:00::","call_ids":["semeval2013.d004.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"amount%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.005085000000000001,"input_uncached_usd":0.0023850000000000004,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.862808334059082}
{"item_id":"semeval2013.d004.s006.t003","gold_sense_keys":["sale%1:04:00::"],"candidates":[{"index":1,"sense_key":"sale%1:04:01::","synset_id":"sale.n.03"},{"index":2,"sense_key":"sale%1:10:00::","synset_id":"sale.n.05"},{"index":3,"sense_key":"sale%1:04:00::","synset_id":"sale.n.01"},{"index":4,"sense_key":"sale%1:26:00::","synset_id":"sale.n.04"},{"index":5,"sense_key":"sale%1:04:02::","synset_id":"sale.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"sale%1:04:00::","call_ids":["semeval2013.d004.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"sale%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":590,"cached_input_tokens":0,"output_tokens":479,"reasoning_output_tokens":464},"cost":{"currency":"USD","total_usd":0.017320000000000002,"input_uncached_usd":0.0029500000000000004,"input_cached_usd":0.0,"output_usd":0.01437,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":15.26443395903334}
{"item_id":"semeval2013.d004.s007.t000","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":2,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":3,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":4,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":5,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":6,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"},{"index":7,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":8,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":9,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":10,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.006280000000000001,"input_uncached_usd":0.00385,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6811608750140294}
{"item_id":"semeval2013.d004.s008.t000","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":2,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":3,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"},{"index":6,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":7,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":8,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":9,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":10,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":756,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.0059700000000000005,"input_uncached_usd":0.0037800000000000004,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.123204999952577}
{"item_id":"semeval2013.d004.s008.t001","gold_sense_keys":["height%1:26:00::"],"candidates":[{"index":1,"sense_key":"height%1:07:02::","synset_id":"altitude.n.01"},{"index":2,"sense_key":"height%1:07:00::","synset_id":"height.n.01"},{"index":3,"sense_key":"height%1:26:00::","synset_id":"acme.n.01"},{"index":4,"sense_key":"height%1:07:01::","synset_id":"stature.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"height%1:26:00::","call_ids":["semeval2013.d004.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"height%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.0046500000000000005,"input_uncached_usd":0.0024600000000000004,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.022438833024353}
{"item_id":"semeval2013.d004.s008.t002","gold_sense_keys":["crisis%1:26:00::"],"candidates":[{"index":1,"sense_key":"crisis%1:11:00::","synset_id":"crisis.n.02"},{"index":2,"sense_key":"crisis%1:26:00::","synset_id":"crisis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"crisis%1:26:00::","call_ids":["semeval2013.d004.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"crisis%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.0036300000000000004,"input_uncached_usd":0.0019500000000000001,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.442994625074789}
{"item_id":"semeval2013.d004.s008.t003","gold_sense_keys":["investment%1:04:00::","investment%1:21:00::"],"candidates":[{"index":1,"sense_key":"investment%1:04:02::","synset_id":"investment.n.06"},{"index":2,"sense_key":"investment%1:21:00::","synset_id":"investment.n.02"},{"index":3,"sense_key":"investment%1:04:00::","synset_id":"investing.n.01"},{"index":4,"sense_key":"investment%1:07:00::","synset_id":"investment.n.03"},{"index":5,"sense_key":"investment%1:04:01::","synset_id":"investment.n.05"},{"index":6,"sense_key":"investment%1:05:00::","synset_id":"investment.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"investment%1:04:00::","call_ids":["semeval2013.d004.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"investment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018575,"input_uncached_usd":0.002735,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.555519917048514}
{"item_id":"semeval2013.d004.s008.t004","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":2,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":3,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":4,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":5,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":6,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":7,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":8,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":9,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":756,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.0056700000000000006,"input_uncached_usd":0.0037800000000000004,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.602580333012156}
{"item_id":"semeval2013.d004.s009.t000","gold_sense_keys":["cash%1:21:00::"],"candidates":[{"index":1,"sense_key":"cash%1:18:00::","synset_id":"cash.n.03"},{"index":2,"sense_key":"cash%1:21:00::","synset_id":"cash.n.01"},{"index":3,"sense_key":"cash%1:21:02::","synset_id":"cash.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cash%1:21:00::","call_ids":["semeval2013.d004.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cash%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":436,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":107},"cost":{"currency":"USD","total_usd":0.00587,"input_uncached_usd":0.00218,"input_cached_usd":0.0,"output_usd":0.00369,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.68920825002715}
{"item_id":"semeval2013.d004.s009.t001","gold_sense_keys":["liability%1:26:00::"],"candidates":[{"index":1,"sense_key":"liability%1:26:01::","synset_id":"liability.n.01"},{"index":2,"sense_key":"liability%1:26:00::","synset_id":"indebtedness.n.01"},{"index":3,"sense_key":"liability%1:07:00::","synset_id":"liability.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"liability%1:26:00::","call_ids":["semeval2013.d004.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"liability%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.004475,"input_uncached_usd":0.0020450000000000004,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.280075583024882}
{"item_id":"semeval2013.d004.s009.t002","gold_sense_keys":["state%1:14:01::"],"candidates":[{"index":1,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"},{"index":2,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":5,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":6,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":7,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":8,"sense_key":"state%1:14:00::","synset_id":"state.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"state%1:14:01::","call_ids":["semeval2013.d004.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"state%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":155,"reasoning_output_tokens":140},"cost":{"currency":"USD","total_usd":0.008199999999999999,"input_uncached_usd":0.00355,"input_cached_usd":0.0,"output_usd":0.0046500000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8969542500562966}
{"item_id":"semeval2013.d004.s010.t002","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":3,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":4,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"},{"index":10,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":734,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.00568,"input_uncached_usd":0.00367,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.234953042003326}
{"item_id":"semeval2013.d004.s010.t003","gold_sense_keys":["capital%1:21:00::"],"candidates":[{"index":1,"sense_key":"capital%1:15:01::","synset_id":"capital.n.05"},{"index":2,"sense_key":"capital%1:15:00::","synset_id":"capital.n.03"},{"index":3,"sense_key":"capital%1:21:00::","synset_id":"capital.n.02"},{"index":4,"sense_key":"capital%1:10:00::","synset_id":"capital.n.04"},{"index":5,"sense_key":"capital%1:14:00::","synset_id":"capital.n.06"},{"index":6,"sense_key":"capital%1:10:01::","synset_id":"das_kapital.n.01"},{"index":7,"sense_key":"capital%1:06:00::","synset_id":"capital.n.08"},{"index":8,"sense_key":"capital%1:21:01::","synset_id":"capital.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"capital%1:21:00::","call_ids":["semeval2013.d004.s010.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"capital%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":210,"reasoning_output_tokens":194},"cost":{"currency":"USD","total_usd":0.009370000000000002,"input_uncached_usd":0.0030700000000000002,"input_cached_usd":0.0,"output_usd":0.0063,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7278295410797}
{"item_id":"semeval2013.d004.s010.t004","gold_sense_keys":["increase%1:07:00::","increase%1:11:00::"],"candidates":[{"index":1,"sense_key":"increase%1:07:00::","synset_id":"increase.n.04"},{"index":2,"sense_key":"increase%1:11:00::","synset_id":"increase.n.02"},{"index":3,"sense_key":"increase%1:22:00::","synset_id":"increase.n.03"},{"index":4,"sense_key":"increase%1:23:00::","synset_id":"addition.n.03"},{"index":5,"sense_key":"increase%1:04:00::","synset_id":"increase.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"increase%1:07:00::","call_ids":["semeval2013.d004.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"increase%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":8108,"reasoning_output_tokens":8002},"cost":{"currency":"USD","total_usd":0.24574,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.24324,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":157.3072895000223}
{"item_id":"semeval2013.d004.s010.t006","gold_sense_keys":["circle%1:14:00::"],"candidates":[{"index":1,"sense_key":"circle%1:25:00::","synset_id":"circle.n.01"},{"index":2,"sense_key":"circle%1:14:00::","synset_id":"set.n.05"},{"index":3,"sense_key":"circle%1:25:01::","synset_id":"circle.n.03"},{"index":4,"sense_key":"circle%1:06:01::","synset_id":"traffic_circle.n.01"},{"index":5,"sense_key":"circle%1:06:02::","synset_id":"circle.n.07"},{"index":6,"sense_key":"circle%1:06:03::","synset_id":"r-2.n.01"},{"index":7,"sense_key":"circle%1:04:00::","synset_id":"lap.n.05"},{"index":8,"sense_key":"circle%1:06:00::","synset_id":"circle.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"circle%1:14:00::","call_ids":["semeval2013.d004.s010.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"circle%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":97},"cost":{"currency":"USD","total_usd":0.006635,"input_uncached_usd":0.003245,"input_cached_usd":0.0,"output_usd":0.0033900000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.347317041014321}
{"item_id":"semeval2013.d004.s012.t000","gold_sense_keys":["support%1:21:01::"],"candidates":[{"index":1,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":2,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":3,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":4,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":7,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":8,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":9,"sense_key":"support%1:21:01::","synset_id":"support.n.11"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:06:01::","synset_id":"support.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"support%1:21:01::","call_ids":["semeval2013.d004.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"support%1:21:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":782,"cached_input_tokens":0,"output_tokens":258,"reasoning_output_tokens":242},"cost":{"currency":"USD","total_usd":0.01165,"input_uncached_usd":0.00391,"input_cached_usd":0.0,"output_usd":0.00774,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.828680083039217}
{"item_id":"semeval2013.d005.s000.t000","gold_sense_keys":["appeal%1:04:00::"],"candidates":[{"index":1,"sense_key":"appeal%1:10:00::","synset_id":"entreaty.n.01"},{"index":2,"sense_key":"appeal%1:07:00::","synset_id":"appeal.n.02"},{"index":3,"sense_key":"appeal%1:10:02::","synset_id":"solicitation.n.02"},{"index":4,"sense_key":"appeal%1:04:00::","synset_id":"appeal.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"appeal%1:04:00::","call_ids":["semeval2013.d005.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"appeal%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":382,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":85},"cost":{"currency":"USD","total_usd":0.00491,"input_uncached_usd":0.0019100000000000002,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.631936750025488}
{"item_id":"semeval2013.d005.s001.t002","gold_sense_keys":["row%1:14:00::"],"candidates":[{"index":1,"sense_key":"row%1:10:00::","synset_id":"quarrel.n.01"},{"index":2,"sense_key":"row%1:14:01::","synset_id":"row.n.05"},{"index":3,"sense_key":"row%1:17:00::","synset_id":"row.n.03"},{"index":4,"sense_key":"row%1:06:00::","synset_id":"course.n.08"},{"index":5,"sense_key":"row%1:07:00::","synset_id":"row.n.06"},{"index":6,"sense_key":"row%1:14:00::","synset_id":"row.n.01"},{"index":7,"sense_key":"row%1:04:00::","synset_id":"rowing.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"row%1:14:00::","call_ids":["semeval2013.d005.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"row%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":431,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":90},"cost":{"currency":"USD","total_usd":0.005335,"input_uncached_usd":0.0021550000000000002,"input_cached_usd":0.0,"output_usd":0.00318,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.516788958921097}
{"item_id":"semeval2013.d005.s002.t000","gold_sense_keys":["report%1:10:00::"],"candidates":[{"index":1,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":2,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":3,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:10:03::","synset_id":"report.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"report%1:10:00::","call_ids":["semeval2013.d005.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"report%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":106},"cost":{"currency":"USD","total_usd":0.006405,"input_uncached_usd":0.002745,"input_cached_usd":0.0,"output_usd":0.00366,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1348241670057178}
{"item_id":"semeval2013.d005.s002.t001","gold_sense_keys":["testimony%1:10:00::"],"candidates":[{"index":1,"sense_key":"testimony%1:10:00::","synset_id":"testimony.n.01"},{"index":2,"sense_key":"testimony%1:10:01::","synset_id":"testimony.n.02"},{"index":3,"sense_key":"testimony%1:10:02::","synset_id":"testimony.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"testimony%1:10:00::","call_ids":["semeval2013.d005.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"testimony%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":320,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.0037,"input_uncached_usd":0.0016,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6702092500636354}
{"item_id":"semeval2013.d005.s003.t000","gold_sense_keys":["in_the_end%4:02:00::"],"candidates":[{"index":1,"sense_key":"in_the_end%4:02:02::","synset_id":"in_the_long_run.r.01"},{"index":2,"sense_key":"in_the_end%4:02:00::","synset_id":"ultimately.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"in_the_end%4:02:00::","call_ids":["semeval2013.d005.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"in_the_end%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":379,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.004985,"input_uncached_usd":0.0018950000000000002,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.373081124969758}
{"item_id":"semeval2013.d005.s003.t001","gold_sense_keys":["defense%1:14:00::"],"candidates":[{"index":1,"sense_key":"defense%1:10:01::","synset_id":"refutation.n.01"},{"index":2,"sense_key":"defense%1:14:03::","synset_id":"defense.n.09"},{"index":3,"sense_key":"defense%1:10:00::","synset_id":"defense.n.04"},{"index":4,"sense_key":"defense%1:14:01::","synset_id":"defense.n.03"},{"index":5,"sense_key":"defense%1:04:03::","synset_id":"defense.n.02"},{"index":6,"sense_key":"defense%1:14:00::","synset_id":"defense.n.07"},{"index":7,"sense_key":"defense%1:14:02::","synset_id":"department_of_defense.n.01"},{"index":8,"sense_key":"defense%1:04:01::","synset_id":"defense.n.12"},{"index":9,"sense_key":"defense%1:04:00::","synset_id":"defense.n.01"},{"index":10,"sense_key":"defense%1:06:00::","synset_id":"defensive_structure.n.01"},{"index":11,"sense_key":"defense%1:22:00::","synset_id":"defense_mechanism.n.01"},{"index":12,"sense_key":"defense%1:04:02::","synset_id":"defense.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"defense%1:14:00::","call_ids":["semeval2013.d005.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"defense%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":870,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.0069900000000000006,"input_uncached_usd":0.004350000000000001,"input_cached_usd":0.0,"output_usd":0.00264,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.059081709012389}
{"item_id":"semeval2013.d005.s003.t002","gold_sense_keys":["message%1:10:01::"],"candidates":[{"index":1,"sense_key":"message%1:10:00::","synset_id":"message.n.02"},{"index":2,"sense_key":"message%1:10:01::","synset_id":"message.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"message%1:10:01::","call_ids":["semeval2013.d005.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"message%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":350,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.00406,"input_uncached_usd":0.00175,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.763795500039123}
{"item_id":"semeval2013.d005.s003.t003","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":2,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":3,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":4,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":5,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"},{"index":6,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.0046300000000000004,"input_uncached_usd":0.00286,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8067060830071568}
{"item_id":"semeval2013.d005.s003.t004","gold_sense_keys":["rape%1:04:00::"],"candidates":[{"index":1,"sense_key":"rape%1:04:01::","synset_id":"rape.n.02"},{"index":2,"sense_key":"rape%1:04:00::","synset_id":"rape.n.03"},{"index":3,"sense_key":"rape%1:20:00::","synset_id":"rape.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"rape%1:04:00::","call_ids":["semeval2013.d005.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"rape%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.003745,"input_uncached_usd":0.0019450000000000001,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.50993120810017}
{"item_id":"semeval2013.d005.s003.t005","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:07::","synset_id":"child.n.04"},{"index":4,"sense_key":"child%1:18:00::","synset_id":"child.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"child%1:18:00::","call_ids":["semeval2013.d005.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"child%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":140},"cost":{"currency":"USD","total_usd":0.00693,"input_uncached_usd":0.0022500000000000003,"input_cached_usd":0.0,"output_usd":0.00468,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.115034791990183}
{"item_id":"semeval2013.d005.s003.t006","gold_sense_keys":["bed%1:06:00::"],"candidates":[{"index":1,"sense_key":"bed%1:06:00::","synset_id":"bed.n.01"},{"index":2,"sense_key":"bed%1:15:00::","synset_id":"seam.n.03"},{"index":3,"sense_key":"bed%1:06:03::","synset_id":"bed.n.07"},{"index":4,"sense_key":"bed%1:06:01::","synset_id":"bed.n.02"},{"index":5,"sense_key":"bed%1:17:00::","synset_id":"bed.n.03"},{"index":6,"sense_key":"bed%1:06:02::","synset_id":"bed.n.08"},{"index":7,"sense_key":"bed%1:17:01::","synset_id":"bed.n.04"},{"index":8,"sense_key":"bed%1:06:04::","synset_id":"layer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bed%1:06:00::","call_ids":["semeval2013.d005.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bed%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.004765,"input_uncached_usd":0.0030250000000000003,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7726259999908507}
{"item_id":"semeval2013.d005.s003.t007","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":2,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":3,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":4,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":5,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":6,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":7,"sense_key":"house%1:06:01::","synset_id":"house.n.12"},{"index":8,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":9,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":10,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":11,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":12,"sense_key":"house%1:14:03::","synset_id":"house.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"house%1:06:00::","call_ids":["semeval2013.d005.s003.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":750,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0054600000000000004,"input_uncached_usd":0.0037500000000000003,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.21036908403039}
{"item_id":"semeval2013.d005.s004.t001","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":2,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":5,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"},{"index":6,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.005,"input_uncached_usd":0.00299,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7923676250502467}
{"item_id":"semeval2013.d005.s004.t002","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":2,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":5,"sense_key":"member%1:14:00::","synset_id":"member.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"member%1:18:00::","call_ids":["semeval2013.d005.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"member%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.004835000000000001,"input_uncached_usd":0.002765,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8420438750181347}
{"item_id":"semeval2013.d005.s004.t003","gold_sense_keys":["family%1:14:00::","family%1:14:02::"],"candidates":[{"index":1,"sense_key":"family%1:18:00::","synset_id":"kin.n.01"},{"index":2,"sense_key":"family%1:14:01::","synset_id":"family.n.04"},{"index":3,"sense_key":"family%1:14:06::","synset_id":"syndicate.n.01"},{"index":4,"sense_key":"family%1:14:02::","synset_id":"family.n.01"},{"index":5,"sense_key":"family%1:14:03::","synset_id":"family.n.06"},{"index":6,"sense_key":"family%1:14:05::","synset_id":"family.n.08"},{"index":7,"sense_key":"family%1:14:04::","synset_id":"class.n.01"},{"index":8,"sense_key":"family%1:14:00::","synset_id":"family.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"family%1:14:00::","call_ids":["semeval2013.d005.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"family%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":310,"reasoning_output_tokens":294},"cost":{"currency":"USD","total_usd":0.012645,"input_uncached_usd":0.0033450000000000003,"input_cached_usd":0.0,"output_usd":0.009300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.972367041977122}
{"item_id":"semeval2013.d005.s004.t004","gold_sense_keys":["end%1:11:00::","end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:18:00::","synset_id":"end.n.08"},{"index":2,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":3,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":4,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"},{"index":5,"sense_key":"end%1:04:02::","synset_id":"end.n.14"},{"index":6,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":7,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":8,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":9,"sense_key":"end%1:15:00::","synset_id":"end.n.01"},{"index":10,"sense_key":"end%1:15:02::","synset_id":"end.n.07"},{"index":11,"sense_key":"end%1:11:00::","synset_id":"end.n.03"},{"index":12,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":13,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":14,"sense_key":"end%1:15:01::","synset_id":"end.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"end%1:28:00::","call_ids":["semeval2013.d005.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"end%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":896,"cached_input_tokens":0,"output_tokens":183,"reasoning_output_tokens":167},"cost":{"currency":"USD","total_usd":0.00997,"input_uncached_usd":0.0044800000000000005,"input_cached_usd":0.0,"output_usd":0.00549,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7173714999808}
{"item_id":"semeval2013.d005.s004.t005","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d005.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":369,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.003255,"input_uncached_usd":0.001845,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.105754915974103}
{"item_id":"semeval2013.d005.s004.t006","gold_sense_keys":["imposition%1:04:02::"],"candidates":[{"index":1,"sense_key":"imposition%1:09:00::","synset_id":"imposition.n.02"},{"index":2,"sense_key":"imposition%1:04:02::","synset_id":"imposition.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"imposition%1:04:02::","call_ids":["semeval2013.d005.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"imposition%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":374,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.00412,"input_uncached_usd":0.0018700000000000001,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6758272919105366}
{"item_id":"semeval2013.d005.s004.t007","gold_sense_keys":["jury%1:14:00::"],"candidates":[{"index":1,"sense_key":"jury%1:14:01::","synset_id":"jury.n.02"},{"index":2,"sense_key":"jury%1:14:00::","synset_id":"jury.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"jury%1:14:00::","call_ids":["semeval2013.d005.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"jury%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":364,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.00332,"input_uncached_usd":0.0018200000000000002,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5611697499407455}
{"item_id":"semeval2013.d005.s005.t002","gold_sense_keys":["filing%1:10:00::"],"candidates":[{"index":1,"sense_key":"filing%1:17:00::","synset_id":"filing.n.02"},{"index":2,"sense_key":"filing%1:04:00::","synset_id":"filing.n.04"},{"index":3,"sense_key":"filing%1:10:00::","synset_id":"filing.n.01"},{"index":4,"sense_key":"filing%1:04:01::","synset_id":"filing.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"filing%1:10:00::","call_ids":["semeval2013.d005.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"filing%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.004895,"input_uncached_usd":0.002465,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7652011249447241}
{"item_id":"semeval2013.d005.s005.t003","gold_sense_keys":["appeal%1:04:00::"],"candidates":[{"index":1,"sense_key":"appeal%1:04:00::","synset_id":"appeal.n.03"},{"index":2,"sense_key":"appeal%1:07:00::","synset_id":"appeal.n.02"},{"index":3,"sense_key":"appeal%1:10:00::","synset_id":"entreaty.n.01"},{"index":4,"sense_key":"appeal%1:10:02::","synset_id":"solicitation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"appeal%1:04:00::","call_ids":["semeval2013.d005.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"appeal%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":85},"cost":{"currency":"USD","total_usd":0.005825,"input_uncached_usd":0.0028250000000000003,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.98429000005126}
{"item_id":"semeval2013.d005.s005.t004","gold_sense_keys":["argument%1:10:02::"],"candidates":[{"index":1,"sense_key":"argument%1:10:04::","synset_id":"argument.n.05"},{"index":2,"sense_key":"argument%1:10:01::","synset_id":"argument.n.04"},{"index":3,"sense_key":"argument%1:09:00::","synset_id":"argument.n.06"},{"index":4,"sense_key":"argument%1:10:02::","synset_id":"argument.n.01"},{"index":5,"sense_key":"argument%1:09:01::","synset_id":"argumentation.n.02"},{"index":6,"sense_key":"argument%1:10:00::","synset_id":"argument.n.03"},{"index":7,"sense_key":"argument%1:10:03::","synset_id":"controversy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"argument%1:09:01::","call_ids":["semeval2013.d005.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"argument%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":123},"cost":{"currency":"USD","total_usd":0.007655,"input_uncached_usd":0.0034850000000000003,"input_cached_usd":0.0,"output_usd":0.00417,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.219249584013596}
{"item_id":"semeval2013.d005.s005.t005","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":2,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":3,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":4,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":5,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":6,"sense_key":"court%1:04:00::","synset_id":"court.n.11"},{"index":7,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":8,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":9,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":10,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":11,"sense_key":"court%1:06:05::","synset_id":"court.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"court%1:06:05::","call_ids":["semeval2013.d005.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"court%1:06:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":786,"cached_input_tokens":0,"output_tokens":2082,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.06639,"input_uncached_usd":0.00393,"input_cached_usd":0.0,"output_usd":0.06246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":29.36010208306834}
{"item_id":"semeval2013.d005.s005.t006","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":2,"sense_key":"year%1:14:00::","synset_id":"class.n.06"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:28:01::","synset_id":"year.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d005.s005.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.005345,"input_uncached_usd":0.002735,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9164817080600187}
{"item_id":"semeval2013.d005.s006.t001","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":2,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":3,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"},{"index":4,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":5,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":6,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.005465,"input_uncached_usd":0.003305,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.103539749979973}
{"item_id":"semeval2013.d005.s006.t003","gold_sense_keys":["use%1:04:00::"],"candidates":[{"index":1,"sense_key":"use%1:04:01::","synset_id":"habit.n.02"},{"index":2,"sense_key":"use%1:04:02::","synset_id":"manipulation.n.01"},{"index":3,"sense_key":"use%1:07:03::","synset_id":"use.n.07"},{"index":4,"sense_key":"use%1:07:00::","synset_id":"function.n.02"},{"index":5,"sense_key":"use%1:22:00::","synset_id":"consumption.n.03"},{"index":6,"sense_key":"use%1:07:02::","synset_id":"use.n.03"},{"index":7,"sense_key":"use%1:04:00::","synset_id":"use.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"use%1:04:00::","call_ids":["semeval2013.d005.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"use%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":722,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.00511,"input_uncached_usd":0.0036100000000000004,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4202511659823358}
{"item_id":"semeval2013.d005.s006.t004","gold_sense_keys":["circus%1:04:01::"],"candidates":[{"index":1,"sense_key":"circus%1:06:00::","synset_id":"circus.n.05"},{"index":2,"sense_key":"circus%1:14:00::","synset_id":"circus.n.01"},{"index":3,"sense_key":"circus%1:06:01::","synset_id":"circus.n.04"},{"index":4,"sense_key":"circus%1:04:00::","synset_id":"circus.n.02"},{"index":5,"sense_key":"circus%1:05:00::","synset_id":"circus.n.06"},{"index":6,"sense_key":"circus%1:04:01::","synset_id":"circus.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"circus%1:04:01::","call_ids":["semeval2013.d005.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"circus%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.004985,"input_uncached_usd":0.003125,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.770305166952312}
{"item_id":"semeval2013.d005.s006.t005","gold_sense_keys":["atmosphere%1:26:01::"],"candidates":[{"index":1,"sense_key":"atmosphere%1:26:01::","synset_id":"atmosphere.n.01"},{"index":2,"sense_key":"atmosphere%1:07:00::","synset_id":"air.n.03"},{"index":3,"sense_key":"atmosphere%1:15:00::","synset_id":"atmosphere.n.03"},{"index":4,"sense_key":"atmosphere%1:26:00::","synset_id":"atmosphere.n.04"},{"index":5,"sense_key":"atmosphere%1:17:00::","synset_id":"atmosphere.n.05"},{"index":6,"sense_key":"atmosphere%1:23:00::","synset_id":"standard_atmosphere.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"atmosphere%1:26:01::","call_ids":["semeval2013.d005.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"atmosphere%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":211,"reasoning_output_tokens":196},"cost":{"currency":"USD","total_usd":0.009475,"input_uncached_usd":0.0031450000000000002,"input_cached_usd":0.0,"output_usd":0.0063300000000000006,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.040951499948278}
{"item_id":"semeval2013.d005.s006.t006","gold_sense_keys":["detail%1:09:00::"],"candidates":[{"index":1,"sense_key":"detail%1:10:00::","synset_id":"detail.n.03"},{"index":2,"sense_key":"detail%1:14:00::","synset_id":"contingent.n.02"},{"index":3,"sense_key":"detail%1:14:01::","synset_id":"detail.n.04"},{"index":4,"sense_key":"detail%1:24:00::","synset_id":"detail.n.02"},{"index":5,"sense_key":"detail%1:09:00::","synset_id":"detail.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"detail%1:09:00::","call_ids":["semeval2013.d005.s006.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"detail%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":558,"cached_input_tokens":0,"output_tokens":170,"reasoning_output_tokens":154},"cost":{"currency":"USD","total_usd":0.007890000000000001,"input_uncached_usd":0.0027900000000000004,"input_cached_usd":0.0,"output_usd":0.0051,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4082871249411255}
{"item_id":"semeval2013.d005.s006.t007","gold_sense_keys":["jury%1:14:00::"],"candidates":[{"index":1,"sense_key":"jury%1:14:01::","synset_id":"jury.n.02"},{"index":2,"sense_key":"jury%1:14:00::","synset_id":"jury.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"jury%1:14:00::","call_ids":["semeval2013.d005.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"jury%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.00391,"input_uncached_usd":0.00214,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6891897090245038}
{"item_id":"semeval2013.d005.s006.t008","gold_sense_keys":["passion%1:12:00::"],"candidates":[{"index":1,"sense_key":"passion%1:07:00::","synset_id":"heat.n.04"},{"index":2,"sense_key":"passion%1:12:02::","synset_id":"passion.n.05"},{"index":3,"sense_key":"passion%1:09:00::","synset_id":"love.n.02"},{"index":4,"sense_key":"passion%1:26:02::","synset_id":"passion.n.07"},{"index":5,"sense_key":"passion%1:26:01::","synset_id":"rage.n.03"},{"index":6,"sense_key":"passion%1:16:00::","synset_id":"mania.n.01"},{"index":7,"sense_key":"passion%1:12:00::","synset_id":"passion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"passion%1:12:00::","call_ids":["semeval2013.d005.s006.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"passion%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":166,"reasoning_output_tokens":150},"cost":{"currency":"USD","total_usd":0.007975000000000001,"input_uncached_usd":0.0029950000000000003,"input_cached_usd":0.0,"output_usd":0.00498,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.003096624976024}
{"item_id":"semeval2013.d005.s007.t000","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"},{"index":2,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":3,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":4,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":5,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":6,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"claim%1:10:02::","call_ids":["semeval2013.d005.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"claim%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.0051,"input_uncached_usd":0.00318,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7232894580811262}
{"item_id":"semeval2013.d005.s007.t001","gold_sense_keys":["appeal%1:04:00::"],"candidates":[{"index":1,"sense_key":"appeal%1:10:00::","synset_id":"entreaty.n.01"},{"index":2,"sense_key":"appeal%1:07:00::","synset_id":"appeal.n.02"},{"index":3,"sense_key":"appeal%1:10:02::","synset_id":"solicitation.n.02"},{"index":4,"sense_key":"appeal%1:04:00::","synset_id":"appeal.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"appeal%1:04:00::","call_ids":["semeval2013.d005.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"appeal%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00486,"input_uncached_usd":0.0030600000000000002,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5723986671073362}
{"item_id":"semeval2013.d005.s007.t002","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d005.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.003775,"input_uncached_usd":0.0023350000000000003,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4550517089664936}
{"item_id":"semeval2013.d005.s007.t003","gold_sense_keys":["question%1:10:01::"],"candidates":[{"index":1,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"},{"index":2,"sense_key":"question%1:10:04::","synset_id":"question.n.06"},{"index":3,"sense_key":"question%1:10:00::","synset_id":"question.n.01"},{"index":4,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":5,"sense_key":"question%1:10:03::","synset_id":"question.n.03"},{"index":6,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"question%1:10:01::","call_ids":["semeval2013.d005.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"question%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.005505,"input_uncached_usd":0.003465,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7447596249403432}
{"item_id":"semeval2013.d005.s007.t005","gold_sense_keys":["rule%1:10:00::"],"candidates":[{"index":1,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":2,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":3,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":4,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":5,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":6,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":7,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":8,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"},{"index":9,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":10,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":11,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":12,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"rule%1:10:00::","call_ids":["semeval2013.d005.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"rule%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":949,"cached_input_tokens":0,"output_tokens":227,"reasoning_output_tokens":211},"cost":{"currency":"USD","total_usd":0.011555000000000001,"input_uncached_usd":0.004745,"input_cached_usd":0.0,"output_usd":0.00681,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.951964750071056}
{"item_id":"semeval2013.d005.s008.t000","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":3,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"},{"index":4,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":5,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":6,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":727,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.005555,"input_uncached_usd":0.003635,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.15150525001809}
{"item_id":"semeval2013.d005.s008.t001","gold_sense_keys":["danger%1:04:00::","danger%1:26:00::"],"candidates":[{"index":1,"sense_key":"danger%1:04:00::","synset_id":"risk.n.02"},{"index":2,"sense_key":"danger%1:26:01::","synset_id":"danger.n.03"},{"index":3,"sense_key":"danger%1:26:00::","synset_id":"danger.n.01"},{"index":4,"sense_key":"danger%1:15:00::","synset_id":"danger.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"danger%1:04:00::","call_ids":["semeval2013.d005.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"danger%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":456,"reasoning_output_tokens":440},"cost":{"currency":"USD","total_usd":0.016635,"input_uncached_usd":0.002955,"input_cached_usd":0.0,"output_usd":0.013680000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":508.1295371669112}
{"item_id":"semeval2013.d005.s008.t002","gold_sense_keys":["inclusion%1:04:00::"],"candidates":[{"index":1,"sense_key":"inclusion%1:26:00::","synset_id":"inclusion.n.01"},{"index":2,"sense_key":"inclusion%1:17:00::","synset_id":"inclusion_body.n.01"},{"index":3,"sense_key":"inclusion%1:24:00::","synset_id":"inclusion.n.02"},{"index":4,"sense_key":"inclusion%1:04:00::","synset_id":"inclusion.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"inclusion%1:04:00::","call_ids":["semeval2013.d005.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"inclusion%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":327,"reasoning_output_tokens":311},"cost":{"currency":"USD","total_usd":0.01266,"input_uncached_usd":0.00285,"input_cached_usd":0.0,"output_usd":0.009810000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.116748915985227}
{"item_id":"semeval2013.d005.s008.t004","gold_sense_keys":["individual%1:03:00::"],"candidates":[{"index":1,"sense_key":"individual%1:18:00::","synset_id":"individual.n.02"},{"index":2,"sense_key":"individual%1:03:00::","synset_id":"person.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"individual%1:03:00::","call_ids":["semeval2013.d005.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"individual%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.004335,"input_uncached_usd":0.002475,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5963867919053882}
{"item_id":"semeval2013.d005.s008.t005","gold_sense_keys":["right%1:07:00::"],"candidates":[{"index":1,"sense_key":"right%1:14:00::","synset_id":"right.n.04"},{"index":2,"sense_key":"right%1:07:00::","synset_id":"right.n.01"},{"index":3,"sense_key":"right%1:08:00::","synset_id":"right.n.05"},{"index":4,"sense_key":"right%1:07:01::","synset_id":"right.n.07"},{"index":5,"sense_key":"right%1:04:00::","synset_id":"right.n.06"},{"index":6,"sense_key":"right%1:06:00::","synset_id":"right_field.n.01"},{"index":7,"sense_key":"right%1:21:00::","synset_id":"right.n.08"},{"index":8,"sense_key":"right%1:15:00::","synset_id":"right.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"right%1:07:00::","call_ids":["semeval2013.d005.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"right%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":822,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":105},"cost":{"currency":"USD","total_usd":0.00774,"input_uncached_usd":0.00411,"input_cached_usd":0.0,"output_usd":0.00363,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3365194579819217}
{"item_id":"semeval2013.d005.s009.t003","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":3,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"},{"index":4,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":5,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":6,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"claim%1:10:02::","call_ids":["semeval2013.d005.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"claim%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.006305,"input_uncached_usd":0.0032150000000000004,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8298534579807892}
{"item_id":"semeval2013.d005.s009.t005","gold_sense_keys":["week%1:28:02::"],"candidates":[{"index":1,"sense_key":"week%1:28:02::","synset_id":"week.n.03"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:00::","synset_id":"week.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"week%1:28:02::","call_ids":["semeval2013.d005.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"week%1:28:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":404,"reasoning_output_tokens":388},"cost":{"currency":"USD","total_usd":0.014690000000000002,"input_uncached_usd":0.0025700000000000002,"input_cached_usd":0.0,"output_usd":0.01212,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.275224457960576}
{"item_id":"semeval2013.d005.s009.t007","gold_sense_keys":["evidence%1:09:00::"],"candidates":[{"index":1,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":2,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":3,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evidence%1:09:00::","call_ids":["semeval2013.d005.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01848,"input_uncached_usd":0.0026400000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.359400667017326}
{"item_id":"semeval2013.d005.s009.t008","gold_sense_keys":["jury%1:14:00::"],"candidates":[{"index":1,"sense_key":"jury%1:14:01::","synset_id":"jury.n.02"},{"index":2,"sense_key":"jury%1:14:00::","synset_id":"jury.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"jury%1:14:00::","call_ids":["semeval2013.d005.s009.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"jury%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.0038150000000000002,"input_uncached_usd":0.0023450000000000003,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.600649542058818}
{"item_id":"semeval2013.d005.s009.t009","gold_sense_keys":["passion%1:12:00::"],"candidates":[{"index":1,"sense_key":"passion%1:26:02::","synset_id":"passion.n.07"},{"index":2,"sense_key":"passion%1:26:01::","synset_id":"rage.n.03"},{"index":3,"sense_key":"passion%1:12:02::","synset_id":"passion.n.05"},{"index":4,"sense_key":"passion%1:09:00::","synset_id":"love.n.02"},{"index":5,"sense_key":"passion%1:07:00::","synset_id":"heat.n.04"},{"index":6,"sense_key":"passion%1:12:00::","synset_id":"passion.n.01"},{"index":7,"sense_key":"passion%1:16:00::","synset_id":"mania.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"passion%1:12:00::","call_ids":["semeval2013.d005.s009.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"passion%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":640,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":123},"cost":{"currency":"USD","total_usd":0.00737,"input_uncached_usd":0.0032,"input_cached_usd":0.0,"output_usd":0.00417,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2508969999616966}
{"item_id":"semeval2013.d005.s010.t000","gold_sense_keys":["last_word%1:10:00::"],"candidates":[{"index":1,"sense_key":"last_word%1:07:00::","synset_id":"chic.n.01"},{"index":2,"sense_key":"last_word%1:10:00::","synset_id":"last_word.n.01"},{"index":3,"sense_key":"last_word%1:10:01::","synset_id":"last_word.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"last_word%1:10:01::","call_ids":["semeval2013.d005.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"last_word%1:10:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":415,"reasoning_output_tokens":399},"cost":{"currency":"USD","total_usd":0.014905,"input_uncached_usd":0.002455,"input_cached_usd":0.0,"output_usd":0.012450000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.346234249998815}
{"item_id":"semeval2013.d005.s010.t001","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"},{"index":8,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":11,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"issue%1:09:01::","call_ids":["semeval2013.d005.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":917,"cached_input_tokens":0,"output_tokens":186,"reasoning_output_tokens":170},"cost":{"currency":"USD","total_usd":0.010165,"input_uncached_usd":0.0045850000000000005,"input_cached_usd":0.0,"output_usd":0.00558,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.072142124990933}
{"item_id":"semeval2013.d005.s011.t000","gold_sense_keys":["decade%1:28:00::"],"candidates":[{"index":1,"sense_key":"decade%1:23:00::","synset_id":"ten.n.01"},{"index":2,"sense_key":"decade%1:28:00::","synset_id":"decade.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"decade%1:28:00::","call_ids":["semeval2013.d005.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"decade%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.004215,"input_uncached_usd":0.002475,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6680206659948453}
{"item_id":"semeval2013.d005.s011.t002","gold_sense_keys":["coverage%1:10:00::"],"candidates":[{"index":1,"sense_key":"coverage%1:10:00::","synset_id":"coverage.n.03"},{"index":2,"sense_key":"coverage%1:07:00::","synset_id":"coverage.n.02"},{"index":3,"sense_key":"coverage%1:21:00::","synset_id":"coverage.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"coverage%1:10:00::","call_ids":["semeval2013.d005.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"coverage%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.00496,"input_uncached_usd":0.00265,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4558796659111977}
{"item_id":"semeval2013.d005.s011.t003","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":2,"sense_key":"case%1:06:01::","synset_id":"case.n.20"},{"index":3,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":4,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":5,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":6,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":7,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":8,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":9,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":10,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":11,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":12,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":13,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":14,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":15,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":16,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":17,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":18,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":19,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":20,"sense_key":"case%1:10:02::","synset_id":"case.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d005.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1277,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.008275000000000001,"input_uncached_usd":0.006385000000000001,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.760380333987996}
{"item_id":"semeval2013.d005.s011.t004","gold_sense_keys":["question%1:07:00::"],"candidates":[{"index":1,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":2,"sense_key":"question%1:10:00::","synset_id":"question.n.01"},{"index":3,"sense_key":"question%1:10:04::","synset_id":"question.n.06"},{"index":4,"sense_key":"question%1:10:03::","synset_id":"question.n.03"},{"index":5,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"},{"index":6,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"question%1:07:00::","call_ids":["semeval2013.d005.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"question%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":709,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":94},"cost":{"currency":"USD","total_usd":0.006845,"input_uncached_usd":0.0035450000000000004,"input_cached_usd":0.0,"output_usd":0.0033,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.625328291906044}
{"item_id":"semeval2013.d005.s011.t005","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":2,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":3,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":4,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":5,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s011.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.005925000000000001,"input_uncached_usd":0.0035550000000000004,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6783285829005763}
{"item_id":"semeval2013.d005.s011.t006","gold_sense_keys":["fairness%1:07:00::"],"candidates":[{"index":1,"sense_key":"fairness%1:07:01::","synset_id":"paleness.n.02"},{"index":2,"sense_key":"fairness%1:07:02::","synset_id":"comeliness.n.01"},{"index":3,"sense_key":"fairness%1:07:00::","synset_id":"fairness.n.01"},{"index":4,"sense_key":"fairness%1:09:00::","synset_id":"fairness.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fairness%1:07:00::","call_ids":["semeval2013.d005.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fairness%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.004765,"input_uncached_usd":0.0028450000000000003,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0796371250180528}
{"item_id":"semeval2013.d005.s012.t002","gold_sense_keys":["conviction%1:04:00::"],"candidates":[{"index":1,"sense_key":"conviction%1:09:00::","synset_id":"conviction.n.01"},{"index":2,"sense_key":"conviction%1:04:00::","synset_id":"conviction.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"conviction%1:04:00::","call_ids":["semeval2013.d005.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"conviction%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.00387,"input_uncached_usd":0.0024300000000000003,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2488665420096368}
{"item_id":"semeval2013.d005.s012.t003","gold_sense_keys":["cleveland%1:15:00::"],"candidates":[{"index":1,"sense_key":"cleveland%1:15:00::","synset_id":"cleveland.n.01"},{"index":2,"sense_key":"cleveland%1:18:00::","synset_id":"cleveland.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cleveland%1:15:00::","call_ids":["semeval2013.d005.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cleveland%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.004645,"input_uncached_usd":0.002395,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7936725419713184}
{"item_id":"semeval2013.d005.s012.t007","gold_sense_keys":["decision%1:09:00::"],"candidates":[{"index":1,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"},{"index":2,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"},{"index":3,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"},{"index":4,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"},{"index":5,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"decision%1:09:00::","call_ids":["semeval2013.d005.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"decision%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.00621,"input_uncached_usd":0.00303,"input_cached_usd":0.0,"output_usd":0.00318,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9485169170657173}
{"item_id":"semeval2013.d005.s012.t008","gold_sense_keys":["carnival%1:04:02::"],"candidates":[{"index":1,"sense_key":"carnival%1:04:00::","synset_id":"carnival.n.01"},{"index":2,"sense_key":"carnival%1:04:02::","synset_id":"circus.n.03"},{"index":3,"sense_key":"carnival%1:04:01::","synset_id":"carnival.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"carnival%1:04:02::","call_ids":["semeval2013.d005.s012.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"carnival%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.004905,"input_uncached_usd":0.002565,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0669009159319103}
{"item_id":"semeval2013.d005.s012.t009","gold_sense_keys":["atmosphere%1:26:01::"],"candidates":[{"index":1,"sense_key":"atmosphere%1:15:00::","synset_id":"atmosphere.n.03"},{"index":2,"sense_key":"atmosphere%1:26:01::","synset_id":"atmosphere.n.01"},{"index":3,"sense_key":"atmosphere%1:07:00::","synset_id":"air.n.03"},{"index":4,"sense_key":"atmosphere%1:17:00::","synset_id":"atmosphere.n.05"},{"index":5,"sense_key":"atmosphere%1:26:00::","synset_id":"atmosphere.n.04"},{"index":6,"sense_key":"atmosphere%1:23:00::","synset_id":"standard_atmosphere.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"atmosphere%1:26:01::","call_ids":["semeval2013.d005.s012.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"atmosphere%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":129},"cost":{"currency":"USD","total_usd":0.007615,"input_uncached_usd":0.003265,"input_cached_usd":0.0,"output_usd":0.00435,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0734678340377286}
{"item_id":"semeval2013.d005.s012.t010","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":2,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"},{"index":3,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":4,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":5,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":6,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s012.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":683,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.0053950000000000005,"input_uncached_usd":0.003415,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8872887920588255}
{"item_id":"semeval2013.d005.s013.t000","gold_sense_keys":["crime%1:04:00::"],"candidates":[{"index":1,"sense_key":"crime%1:04:00::","synset_id":"crime.n.01"},{"index":2,"sense_key":"crime%1:04:01::","synset_id":"crime.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crime%1:04:00::","call_ids":["semeval2013.d005.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crime%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.0048400000000000006,"input_uncached_usd":0.00253,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.054788542096503}
{"item_id":"semeval2013.d005.s013.t001","gold_sense_keys":["inspiration%1:09:02::"],"candidates":[{"index":1,"sense_key":"inspiration%1:09:01::","synset_id":"inspiration.n.03"},{"index":2,"sense_key":"inspiration%1:09:00::","synset_id":"divine_guidance.n.01"},{"index":3,"sense_key":"inspiration%1:09:02::","synset_id":"inspiration.n.01"},{"index":4,"sense_key":"inspiration%1:04:00::","synset_id":"inhalation.n.01"},{"index":5,"sense_key":"inspiration%1:04:01::","synset_id":"inspiration.n.05"},{"index":6,"sense_key":"inspiration%1:06:00::","synset_id":"inspiration.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"inspiration%1:09:02::","call_ids":["semeval2013.d005.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"inspiration%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":118},"cost":{"currency":"USD","total_usd":0.0073100000000000005,"input_uncached_usd":0.0032900000000000004,"input_cached_usd":0.0,"output_usd":0.00402,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.851545082987286}
{"item_id":"semeval2013.d005.s014.t002","gold_sense_keys":["conviction%1:04:00::"],"candidates":[{"index":1,"sense_key":"conviction%1:09:00::","synset_id":"conviction.n.01"},{"index":2,"sense_key":"conviction%1:04:00::","synset_id":"conviction.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"conviction%1:04:00::","call_ids":["semeval2013.d005.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"conviction%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.004285000000000001,"input_uncached_usd":0.002605,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0217604159843177}
{"item_id":"semeval2013.d005.s014.t003","gold_sense_keys":["half%1:23:00::"],"candidates":[{"index":1,"sense_key":"half%1:28:00::","synset_id":"half.n.02"},{"index":2,"sense_key":"half%1:23:00::","synset_id":"one-half.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"half%1:23:00::","call_ids":["semeval2013.d005.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"half%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.0047,"input_uncached_usd":0.00251,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0113067500060424}
{"item_id":"semeval2013.d005.s014.t004","gold_sense_keys":["century%1:28:00::"],"candidates":[{"index":1,"sense_key":"century%1:28:00::","synset_id":"century.n.01"},{"index":2,"sense_key":"century%1:23:00::","synset_id":"hundred.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"century%1:28:00::","call_ids":["semeval2013.d005.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"century%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.0043300000000000005,"input_uncached_usd":0.0024100000000000002,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9525030839722604}
{"item_id":"semeval2013.d005.s014.t005","gold_sense_keys":["argument%1:09:01::","argument%1:10:02::"],"candidates":[{"index":1,"sense_key":"argument%1:09:00::","synset_id":"argument.n.06"},{"index":2,"sense_key":"argument%1:10:01::","synset_id":"argument.n.04"},{"index":3,"sense_key":"argument%1:10:02::","synset_id":"argument.n.01"},{"index":4,"sense_key":"argument%1:09:01::","synset_id":"argumentation.n.02"},{"index":5,"sense_key":"argument%1:10:03::","synset_id":"controversy.n.01"},{"index":6,"sense_key":"argument%1:10:04::","synset_id":"argument.n.05"},{"index":7,"sense_key":"argument%1:10:00::","synset_id":"argument.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"argument%1:09:01::","call_ids":["semeval2013.d005.s014.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"argument%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":768,"cached_input_tokens":0,"output_tokens":1819,"reasoning_output_tokens":1786},"cost":{"currency":"USD","total_usd":0.058410000000000004,"input_uncached_usd":0.0038400000000000005,"input_cached_usd":0.0,"output_usd":0.05457,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":25.41794087504968}
{"item_id":"semeval2013.d005.s014.t006","gold_sense_keys":["rule%1:10:00::"],"candidates":[{"index":1,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":2,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":3,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":4,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":5,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":6,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":7,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"},{"index":8,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":9,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":10,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":11,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":12,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"rule%1:10:00::","call_ids":["semeval2013.d005.s014.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"rule%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":974,"cached_input_tokens":0,"output_tokens":305,"reasoning_output_tokens":289},"cost":{"currency":"USD","total_usd":0.014020000000000001,"input_uncached_usd":0.00487,"input_cached_usd":0.0,"output_usd":0.00915,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.729806832969189}
{"item_id":"semeval2013.d005.s014.t007","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":2,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":3,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":6,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":7,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":8,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":9,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":10,"sense_key":"way%1:21:00::","synset_id":"way.n.12"},{"index":11,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":12,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"way%1:04:01::","call_ids":["semeval2013.d005.s014.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":894,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02031,"input_uncached_usd":0.00447,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.146260708919726}
{"item_id":"semeval2013.d005.s014.t010","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":2,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":3,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":4,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":5,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":6,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":7,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":10,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":11,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":12,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":13,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":14,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":15,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":16,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":17,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":18,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":19,"sense_key":"case%1:06:01::","synset_id":"case.n.20"},{"index":20,"sense_key":"case%1:23:00::","synset_id":"case.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d005.s014.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1284,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":113},"cost":{"currency":"USD","total_usd":0.01029,"input_uncached_usd":0.00642,"input_cached_usd":0.0,"output_usd":0.00387,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.153183416929096}
{"item_id":"semeval2013.d005.s015.t000","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::","reason%1:16:00::"],"candidates":[{"index":1,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":2,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":3,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":4,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":5,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"},{"index":6,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"reason%1:10:00::","call_ids":["semeval2013.d005.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"reason%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":683,"cached_input_tokens":0,"output_tokens":337,"reasoning_output_tokens":322},"cost":{"currency":"USD","total_usd":0.013525,"input_uncached_usd":0.003415,"input_cached_usd":0.0,"output_usd":0.010110000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.176169749931432}
{"item_id":"semeval2013.d005.s015.t001","gold_sense_keys":["concern%1:12:01::"],"candidates":[{"index":1,"sense_key":"concern%1:09:00::","synset_id":"concern.n.01"},{"index":2,"sense_key":"concern%1:12:00::","synset_id":"concern.n.03"},{"index":3,"sense_key":"concern%1:09:01::","synset_id":"concern.n.04"},{"index":4,"sense_key":"concern%1:14:00::","synset_id":"business.n.01"},{"index":5,"sense_key":"concern%1:12:01::","synset_id":"concern.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"concern%1:12:01::","call_ids":["semeval2013.d005.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"concern%1:12:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":267,"reasoning_output_tokens":251},"cost":{"currency":"USD","total_usd":0.0112,"input_uncached_usd":0.00319,"input_cached_usd":0.0,"output_usd":0.00801,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.978133541997522}
{"item_id":"semeval2013.d005.s015.t002","gold_sense_keys":["impact%1:19:00::"],"candidates":[{"index":1,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":2,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"},{"index":3,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":4,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"impact%1:19:00::","call_ids":["semeval2013.d005.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"impact%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":562,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.005900000000000001,"input_uncached_usd":0.0028100000000000004,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0615784160327166}
{"item_id":"semeval2013.d005.s015.t003","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":2,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":3,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"},{"index":4,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":5,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":6,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":39,"reasoning_output_tokens":24},"cost":{"currency":"USD","total_usd":0.004660000000000001,"input_uncached_usd":0.0034900000000000005,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":48.00876754196361}
{"item_id":"semeval2013.d005.s015.t005","gold_sense_keys":["connecticut%1:15:00::"],"candidates":[{"index":1,"sense_key":"connecticut%1:15:01::","synset_id":"connecticut.n.03"},{"index":2,"sense_key":"connecticut%1:17:00::","synset_id":"connecticut.n.02"},{"index":3,"sense_key":"connecticut%1:15:00::","synset_id":"connecticut.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"connecticut%1:15:00::","call_ids":["semeval2013.d005.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"connecticut%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.00436,"input_uncached_usd":0.00265,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.911892500007525}
{"item_id":"semeval2013.d005.s015.t007","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":2,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":3,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":4,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":5,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":6,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2013.d005.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":727,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.005855000000000001,"input_uncached_usd":0.003635,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6389304170152172}
{"item_id":"semeval2013.d005.s016.t001","gold_sense_keys":["appeal%1:04:00::"],"candidates":[{"index":1,"sense_key":"appeal%1:10:00::","synset_id":"entreaty.n.01"},{"index":2,"sense_key":"appeal%1:10:02::","synset_id":"solicitation.n.02"},{"index":3,"sense_key":"appeal%1:07:00::","synset_id":"appeal.n.02"},{"index":4,"sense_key":"appeal%1:04:00::","synset_id":"appeal.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"appeal%1:04:00::","call_ids":["semeval2013.d005.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"appeal%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.005185,"input_uncached_usd":0.0031450000000000002,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0777437910437584}
{"item_id":"semeval2013.d005.s016.t002","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d005.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.004490000000000001,"input_uncached_usd":0.0024200000000000003,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7748636660398915}
{"item_id":"semeval2013.d005.s016.t003","gold_sense_keys":["idea%1:09:00::"],"candidates":[{"index":1,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"},{"index":2,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"},{"index":3,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"},{"index":4,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"},{"index":5,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"idea%1:09:00::","call_ids":["semeval2013.d005.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"idea%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":660,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":134},"cost":{"currency":"USD","total_usd":0.007800000000000001,"input_uncached_usd":0.0033000000000000004,"input_cached_usd":0.0,"output_usd":0.0045000000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.546711583971046}
{"item_id":"semeval2013.d005.s016.t004","gold_sense_keys":["communication%1:10:01::"],"candidates":[{"index":1,"sense_key":"communication%1:03:00::","synset_id":"communication.n.02"},{"index":2,"sense_key":"communication%1:10:01::","synset_id":"communication.n.01"},{"index":3,"sense_key":"communication%1:24:00::","synset_id":"communication.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"communication%1:10:01::","call_ids":["semeval2013.d005.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"communication%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":539,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.005305000000000001,"input_uncached_usd":0.0026950000000000003,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2692404580302536}
{"item_id":"semeval2013.d005.s016.t006","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":6,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":713,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.005215000000000001,"input_uncached_usd":0.0035650000000000005,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6641768330009654}
{"item_id":"semeval2013.d005.s017.t000","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":2,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":3,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"},{"index":8,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":11,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"issue%1:09:01::","call_ids":["semeval2013.d005.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":983,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":135},"cost":{"currency":"USD","total_usd":0.009415,"input_uncached_usd":0.004915,"input_cached_usd":0.0,"output_usd":0.0045000000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2853979589417577}
{"item_id":"semeval2013.d005.s017.t002","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:14:00::","synset_id":"class.n.06"},{"index":2,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":3,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":4,"sense_key":"year%1:28:02::","synset_id":"year.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d005.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":635,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.005095000000000001,"input_uncached_usd":0.0031750000000000003,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.010351708973758}
{"item_id":"semeval2013.d005.s017.t003","gold_sense_keys":["execution%1:04:00::"],"candidates":[{"index":1,"sense_key":"execution%1:04:01::","synset_id":"murder.n.01"},{"index":2,"sense_key":"execution%1:22:00::","synset_id":"execution.n.03"},{"index":3,"sense_key":"execution%1:04:03::","synset_id":"execution.n.06"},{"index":4,"sense_key":"execution%1:10:00::","synset_id":"execution.n.05"},{"index":5,"sense_key":"execution%1:04:00::","synset_id":"execution.n.01"},{"index":6,"sense_key":"execution%1:10:01::","synset_id":"execution.n.04"},{"index":7,"sense_key":"execution%1:04:02::","synset_id":"performance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"execution%1:04:00::","call_ids":["semeval2013.d005.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"execution%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.0056700000000000006,"input_uncached_usd":0.0038100000000000005,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6536982910474762}
{"item_id":"semeval2013.d005.s018.t002","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"},{"index":2,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d005.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.003895,"input_uncached_usd":0.002425,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.662755790981464}
{"item_id":"semeval2013.d005.s018.t004","gold_sense_keys":["reaction%1:04:00::","reaction%1:09:00::","reaction%1:19:01::"],"candidates":[{"index":1,"sense_key":"reaction%1:19:01::","synset_id":"reaction.n.05"},{"index":2,"sense_key":"reaction%1:19:00::","synset_id":"reaction.n.04"},{"index":3,"sense_key":"reaction%1:04:01::","synset_id":"reaction.n.07"},{"index":4,"sense_key":"reaction%1:09:01::","synset_id":"reaction.n.06"},{"index":5,"sense_key":"reaction%1:22:00::","synset_id":"chemical_reaction.n.01"},{"index":6,"sense_key":"reaction%1:04:00::","synset_id":"reaction.n.03"},{"index":7,"sense_key":"reaction%1:09:00::","synset_id":"reaction.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reaction%1:19:01::","call_ids":["semeval2013.d005.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reaction%1:19:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":98},"cost":{"currency":"USD","total_usd":0.00711,"input_uncached_usd":0.00369,"input_cached_usd":0.0,"output_usd":0.0034200000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7015820000087842}
{"item_id":"semeval2013.d005.s019.t002","gold_sense_keys":["victim%1:18:00::"],"candidates":[{"index":1,"sense_key":"victim%1:18:01::","synset_id":"victim.n.02"},{"index":2,"sense_key":"victim%1:18:00::","synset_id":"victim.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"victim%1:18:00::","call_ids":["semeval2013.d005.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"victim%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00436,"input_uncached_usd":0.00253,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8918900419957936}
{"item_id":"semeval2013.d005.s020.t002","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d005.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.0037800000000000004,"input_uncached_usd":0.00237,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.764771708054468}
{"item_id":"semeval2013.d005.s020.t005","gold_sense_keys":["pressure%1:07:00::"],"candidates":[{"index":1,"sense_key":"pressure%1:07:00::","synset_id":"pressure.n.02"},{"index":2,"sense_key":"pressure%1:26:00::","synset_id":"imperativeness.n.01"},{"index":3,"sense_key":"pressure%1:26:01::","synset_id":"pressure.n.06"},{"index":4,"sense_key":"pressure%1:19:00::","synset_id":"pressure.n.01"},{"index":5,"sense_key":"pressure%1:19:01::","synset_id":"atmospheric_pressure.n.01"},{"index":6,"sense_key":"pressure%1:09:00::","synset_id":"pressure.n.05"},{"index":7,"sense_key":"pressure%1:04:00::","synset_id":"press.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pressure%1:07:00::","call_ids":["semeval2013.d005.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pressure%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":722,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":134},"cost":{"currency":"USD","total_usd":0.008110000000000001,"input_uncached_usd":0.0036100000000000004,"input_cached_usd":0.0,"output_usd":0.0045000000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2658777920296416}
{"item_id":"semeval2013.d005.s020.t007","gold_sense_keys":["decision%1:09:00::"],"candidates":[{"index":1,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"},{"index":2,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"},{"index":3,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"},{"index":4,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"},{"index":5,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"decision%1:09:00::","call_ids":["semeval2013.d005.s020.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"decision%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":1897,"reasoning_output_tokens":1865},"cost":{"currency":"USD","total_usd":0.06004,"input_uncached_usd":0.0031300000000000004,"input_cached_usd":0.0,"output_usd":0.05691,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":30.166466208989732}
{"item_id":"semeval2013.d005.s020.t008","gold_sense_keys":["jury%1:14:00::"],"candidates":[{"index":1,"sense_key":"jury%1:14:00::","synset_id":"jury.n.01"},{"index":2,"sense_key":"jury%1:14:01::","synset_id":"jury.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"jury%1:14:00::","call_ids":["semeval2013.d005.s020.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"jury%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.00406,"input_uncached_usd":0.00235,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6773703750222921}
{"item_id":"semeval2013.d005.s022.t000","gold_sense_keys":["argument%1:09:01::","argument%1:10:02::"],"candidates":[{"index":1,"sense_key":"argument%1:10:00::","synset_id":"argument.n.03"},{"index":2,"sense_key":"argument%1:10:02::","synset_id":"argument.n.01"},{"index":3,"sense_key":"argument%1:10:04::","synset_id":"argument.n.05"},{"index":4,"sense_key":"argument%1:09:01::","synset_id":"argumentation.n.02"},{"index":5,"sense_key":"argument%1:10:01::","synset_id":"argument.n.04"},{"index":6,"sense_key":"argument%1:10:03::","synset_id":"controversy.n.01"},{"index":7,"sense_key":"argument%1:09:00::","synset_id":"argument.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"argument%1:09:01::","call_ids":["semeval2013.d005.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"argument%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01948,"input_uncached_usd":0.0036400000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.396915375022218}
{"item_id":"semeval2013.d005.s022.t001","gold_sense_keys":["effect%1:19:00::"],"candidates":[{"index":1,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"},{"index":2,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"},{"index":3,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":4,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"},{"index":5,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":6,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"effect%1:19:00::","call_ids":["semeval2013.d005.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"effect%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":644,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.005560000000000001,"input_uncached_usd":0.00322,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9949053750606254}
{"item_id":"semeval2013.d005.s022.t002","gold_sense_keys":["coverage%1:10:00::"],"candidates":[{"index":1,"sense_key":"coverage%1:07:00::","synset_id":"coverage.n.02"},{"index":2,"sense_key":"coverage%1:10:00::","synset_id":"coverage.n.03"},{"index":3,"sense_key":"coverage%1:21:00::","synset_id":"coverage.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"coverage%1:10:00::","call_ids":["semeval2013.d005.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"coverage%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.00405,"input_uncached_usd":0.00249,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4161919170292094}
{"item_id":"semeval2013.d005.s022.t003","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"},{"index":2,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":5,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":6,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":679,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.005195,"input_uncached_usd":0.0033950000000000004,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":79.43282579106744}
{"item_id":"semeval2013.d005.s022.t004","gold_sense_keys":["interest%1:09:00::"],"candidates":[{"index":1,"sense_key":"interest%1:14:00::","synset_id":"interest.n.06"},{"index":2,"sense_key":"interest%1:09:00::","synset_id":"interest.n.01"},{"index":3,"sense_key":"interest%1:21:03::","synset_id":"interest.n.05"},{"index":4,"sense_key":"interest%1:07:02::","synset_id":"interest.n.03"},{"index":5,"sense_key":"interest%1:07:01::","synset_id":"sake.n.01"},{"index":6,"sense_key":"interest%1:04:01::","synset_id":"pastime.n.01"},{"index":7,"sense_key":"interest%1:21:00::","synset_id":"interest.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"interest%1:09:00::","call_ids":["semeval2013.d005.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"interest%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019395,"input_uncached_usd":0.0035550000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.620807333034463}
{"item_id":"semeval2013.d005.s023.t000","gold_sense_keys":["challenge%1:10:01::"],"candidates":[{"index":1,"sense_key":"challenge%1:26:00::","synset_id":"challenge.n.01"},{"index":2,"sense_key":"challenge%1:10:01::","synset_id":"challenge.n.03"},{"index":3,"sense_key":"challenge%1:10:00::","synset_id":"challenge.n.02"},{"index":4,"sense_key":"challenge%1:10:02::","synset_id":"challenge.n.04"},{"index":5,"sense_key":"challenge%1:10:03::","synset_id":"challenge.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"challenge%1:10:01::","call_ids":["semeval2013.d005.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"challenge%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":107},"cost":{"currency":"USD","total_usd":0.006325000000000001,"input_uncached_usd":0.002635,"input_cached_usd":0.0,"output_usd":0.00369,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7853421670151874}
{"item_id":"semeval2013.d005.s023.t002","gold_sense_keys":["month%1:28:01::"],"candidates":[{"index":1,"sense_key":"month%1:28:00::","synset_id":"month.n.02"},{"index":2,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:01::","call_ids":["semeval2013.d005.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0041600000000000005,"input_uncached_usd":0.0022400000000000002,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7715807080967352}
{"item_id":"semeval2013.d005.s023.t003","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":4,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":5,"sense_key":"case%1:06:01::","synset_id":"case.n.20"},{"index":6,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":7,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":8,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":9,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":10,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":11,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":12,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":13,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":14,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":15,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":16,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":17,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":18,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":19,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":20,"sense_key":"case%1:10:03::","synset_id":"font.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d005.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1234,"cached_input_tokens":0,"output_tokens":234,"reasoning_output_tokens":219},"cost":{"currency":"USD","total_usd":0.01319,"input_uncached_usd":0.00617,"input_cached_usd":0.0,"output_usd":0.00702,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.201679249992594}
{"item_id":"semeval2013.d005.s023.t004","gold_sense_keys":["news%1:10:01::","news%1:10:02::"],"candidates":[{"index":1,"sense_key":"news%1:10:03::","synset_id":"news.n.04"},{"index":2,"sense_key":"news%1:10:00::","synset_id":"news.n.01"},{"index":3,"sense_key":"news%1:10:01::","synset_id":"news.n.02"},{"index":4,"sense_key":"news%1:07:00::","synset_id":"newsworthiness.n.01"},{"index":5,"sense_key":"news%1:10:02::","synset_id":"news_program.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"news%1:10:00::","call_ids":["semeval2013.d005.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"news%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":593,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018805,"input_uncached_usd":0.002965,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.899256499949843}
{"item_id":"semeval2013.d005.s023.t005","gold_sense_keys":["connecticut%1:15:00::"],"candidates":[{"index":1,"sense_key":"connecticut%1:15:00::","synset_id":"connecticut.n.01"},{"index":2,"sense_key":"connecticut%1:15:01::","synset_id":"connecticut.n.03"},{"index":3,"sense_key":"connecticut%1:17:00::","synset_id":"connecticut.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"connecticut%1:15:00::","call_ids":["semeval2013.d005.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"connecticut%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.004030000000000001,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3724118339596316}
{"item_id":"semeval2013.d005.s023.t006","gold_sense_keys":["election%1:04:01::"],"candidates":[{"index":1,"sense_key":"election%1:09:00::","synset_id":"election.n.04"},{"index":2,"sense_key":"election%1:04:00::","synset_id":"election.n.02"},{"index":3,"sense_key":"election%1:26:00::","synset_id":"election.n.03"},{"index":4,"sense_key":"election%1:04:01::","synset_id":"election.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"election%1:04:01::","call_ids":["semeval2013.d005.s023.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"election%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.004515,"input_uncached_usd":0.002595,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9155394160188735}
{"item_id":"semeval2013.d005.s023.t007","gold_sense_keys":["campaign%1:11:00::"],"candidates":[{"index":1,"sense_key":"campaign%1:04:02::","synset_id":"campaign.n.02"},{"index":2,"sense_key":"campaign%1:04:00::","synset_id":"campaign.n.03"},{"index":3,"sense_key":"campaign%1:04:01::","synset_id":"campaign.n.04"},{"index":4,"sense_key":"campaign%1:11:00::","synset_id":"political_campaign.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"campaign%1:11:00::","call_ids":["semeval2013.d005.s023.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"campaign%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.005180000000000001,"input_uncached_usd":0.0027800000000000004,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":75.81654370797332}
{"item_id":"semeval2013.d005.s023.t008","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["semeval2013.d005.s023.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.00404,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5842287909472361}
{"item_id":"semeval2013.d005.s024.t002","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":2,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":3,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":4,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":5,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":6,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":7,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":8,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":11,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":12,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":13,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":14,"sense_key":"case%1:06:01::","synset_id":"case.n.20"},{"index":15,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":16,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":17,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":18,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":19,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":20,"sense_key":"case%1:18:04::","synset_id":"character.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d005.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1221,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":94},"cost":{"currency":"USD","total_usd":0.009375,"input_uncached_usd":0.006105,"input_cached_usd":0.0,"output_usd":0.00327,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6998952500289306}
{"item_id":"semeval2013.d005.s025.t003","gold_sense_keys":["courthouse%1:06:00::"],"candidates":[{"index":1,"sense_key":"courthouse%1:06:01::","synset_id":"courthouse.n.01"},{"index":2,"sense_key":"courthouse%1:06:00::","synset_id":"courthouse.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"courthouse%1:06:00::","call_ids":["semeval2013.d005.s025.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"courthouse%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":377,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.004255,"input_uncached_usd":0.0018850000000000002,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.469540041987784}
{"item_id":"semeval2013.d005.s026.t000","gold_sense_keys":["focus%1:09:00::"],"candidates":[{"index":1,"sense_key":"focus%1:26:00::","synset_id":"focus.n.04"},{"index":2,"sense_key":"focus%1:26:01::","synset_id":"stress.n.03"},{"index":3,"sense_key":"focus%1:07:03::","synset_id":"focus.n.03"},{"index":4,"sense_key":"focus%1:15:00::","synset_id":"focus.n.07"},{"index":5,"sense_key":"focus%1:19:00::","synset_id":"focus.n.06"},{"index":6,"sense_key":"focus%1:09:00::","synset_id":"focus.n.01"},{"index":7,"sense_key":"focus%1:07:01::","synset_id":"focus.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"focus%1:09:00::","call_ids":["semeval2013.d005.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"focus%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018815,"input_uncached_usd":0.002975,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":17.141227582935244}
{"item_id":"semeval2013.d005.s026.t001","gold_sense_keys":["coverage%1:10:00::"],"candidates":[{"index":1,"sense_key":"coverage%1:21:00::","synset_id":"coverage.n.01"},{"index":2,"sense_key":"coverage%1:10:00::","synset_id":"coverage.n.03"},{"index":3,"sense_key":"coverage%1:07:00::","synset_id":"coverage.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"coverage%1:10:00::","call_ids":["semeval2013.d005.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"coverage%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.005090000000000001,"input_uncached_usd":0.0021200000000000004,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6677899580681697}
{"item_id":"semeval2013.d005.s027.t000","gold_sense_keys":["interview%1:10:01::"],"candidates":[{"index":1,"sense_key":"interview%1:10:01::","synset_id":"interview.n.01"},{"index":2,"sense_key":"interview%1:10:00::","synset_id":"consultation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"interview%1:10:01::","call_ids":["semeval2013.d005.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"interview%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":119},"cost":{"currency":"USD","total_usd":0.00613,"input_uncached_usd":0.0021100000000000003,"input_cached_usd":0.0,"output_usd":0.00402,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3786435410147533}
{"item_id":"semeval2013.d005.s027.t001","gold_sense_keys":["journalist%1:18:00::"],"candidates":[{"index":1,"sense_key":"journalist%1:18:00::","synset_id":"journalist.n.01"},{"index":2,"sense_key":"journalist%1:18:01::","synset_id":"diarist.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"journalist%1:18:00::","call_ids":["semeval2013.d005.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"journalist%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":380,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.00349,"input_uncached_usd":0.0019000000000000002,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.710573749965988}
{"item_id":"semeval2013.d005.s027.t002","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":2,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":3,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"},{"index":4,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":5,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":6,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"claim%1:10:02::","call_ids":["semeval2013.d005.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"claim%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.004955,"input_uncached_usd":0.002795,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6434126250678673}
{"item_id":"semeval2013.d005.s027.t003","gold_sense_keys":["news%1:10:00::","news%1:10:01::","news%1:10:03::"],"candidates":[{"index":1,"sense_key":"news%1:10:02::","synset_id":"news_program.n.01"},{"index":2,"sense_key":"news%1:07:00::","synset_id":"newsworthiness.n.01"},{"index":3,"sense_key":"news%1:10:01::","synset_id":"news.n.02"},{"index":4,"sense_key":"news%1:10:00::","synset_id":"news.n.01"},{"index":5,"sense_key":"news%1:10:03::","synset_id":"news.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"news%1:10:00::","call_ids":["semeval2013.d005.s027.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"news%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":180,"reasoning_output_tokens":164},"cost":{"currency":"USD","total_usd":0.008115,"input_uncached_usd":0.0027150000000000004,"input_cached_usd":0.0,"output_usd":0.0054,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4910904999123886}
{"item_id":"semeval2013.d005.s027.t004","gold_sense_keys":["report%1:10:00::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"},{"index":4,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":5,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":6,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":7,"sense_key":"report%1:11:00::","synset_id":"report.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"report%1:10:00::","call_ids":["semeval2013.d005.s027.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"report%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.005605000000000001,"input_uncached_usd":0.0032350000000000005,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0397338329348713}
{"item_id":"semeval2013.d005.s027.t005","gold_sense_keys":["view%1:09:02::","view%1:09:04::","view%1:10:00::"],"candidates":[{"index":1,"sense_key":"view%1:09:01::","synset_id":"view.n.02"},{"index":2,"sense_key":"view%1:09:00::","synset_id":"view.n.07"},{"index":3,"sense_key":"view%1:07:01::","synset_id":"view.n.10"},{"index":4,"sense_key":"view%1:09:02::","synset_id":"position.n.03"},{"index":5,"sense_key":"view%1:04:00::","synset_id":"view.n.03"},{"index":6,"sense_key":"view%1:06:00::","synset_id":"scene.n.08"},{"index":7,"sense_key":"view%1:10:00::","synset_id":"opinion.n.02"},{"index":8,"sense_key":"view%1:07:00::","synset_id":"horizon.n.02"},{"index":9,"sense_key":"view%1:09:04::","synset_id":"opinion.n.01"},{"index":10,"sense_key":"view%1:15:00::","synset_id":"view.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"view%1:09:02::","call_ids":["semeval2013.d005.s027.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"view%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":1231,"reasoning_output_tokens":1201},"cost":{"currency":"USD","total_usd":0.040874999999999995,"input_uncached_usd":0.003945000000000001,"input_cached_usd":0.0,"output_usd":0.03693,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":19.579379792092368}
{"item_id":"semeval2013.d005.s027.t006","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":2,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":3,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":4,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":5,"sense_key":"court%1:04:00::","synset_id":"court.n.11"},{"index":6,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":7,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":10,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":11,"sense_key":"court%1:06:04::","synset_id":"court.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"court%1:14:00::","call_ids":["semeval2013.d005.s027.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":755,"cached_input_tokens":0,"output_tokens":10816,"reasoning_output_tokens":10678},"cost":{"currency":"USD","total_usd":0.32825499999999996,"input_uncached_usd":0.003775,"input_cached_usd":0.0,"output_usd":0.32448,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":173.44108199991751}
{"item_id":"semeval2013.d005.s027.t007","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"policy%1:09:00::","call_ids":["semeval2013.d005.s027.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":111},"cost":{"currency":"USD","total_usd":0.00604,"input_uncached_usd":0.0022600000000000003,"input_cached_usd":0.0,"output_usd":0.00378,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.301021917024627}
{"item_id":"semeval2013.d005.s027.t008","gold_sense_keys":["coverage%1:10:00::"],"candidates":[{"index":1,"sense_key":"coverage%1:10:00::","synset_id":"coverage.n.03"},{"index":2,"sense_key":"coverage%1:21:00::","synset_id":"coverage.n.01"},{"index":3,"sense_key":"coverage%1:07:00::","synset_id":"coverage.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"coverage%1:10:00::","call_ids":["semeval2013.d005.s027.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"coverage%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.004645,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.138581040897407}
{"item_id":"semeval2013.d005.s027.t009","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":2,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":3,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":4,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":5,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s027.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.00447,"input_uncached_usd":0.0030900000000000003,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3858975840266794}
{"item_id":"semeval2013.d005.s028.t002","gold_sense_keys":["argument%1:09:01::","argument%1:10:02::"],"candidates":[{"index":1,"sense_key":"argument%1:10:03::","synset_id":"controversy.n.01"},{"index":2,"sense_key":"argument%1:09:01::","synset_id":"argumentation.n.02"},{"index":3,"sense_key":"argument%1:10:02::","synset_id":"argument.n.01"},{"index":4,"sense_key":"argument%1:10:04::","synset_id":"argument.n.05"},{"index":5,"sense_key":"argument%1:09:00::","synset_id":"argument.n.06"},{"index":6,"sense_key":"argument%1:10:00::","synset_id":"argument.n.03"},{"index":7,"sense_key":"argument%1:10:01::","synset_id":"argument.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"argument%1:09:01::","call_ids":["semeval2013.d005.s028.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"argument%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01913,"input_uncached_usd":0.0032900000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.656206749961711}
{"item_id":"semeval2013.d005.s028.t003","gold_sense_keys":["change%1:04:00::","change%1:11:00::","change%1:24:00::"],"candidates":[{"index":1,"sense_key":"change%1:21:01::","synset_id":"change.n.08"},{"index":2,"sense_key":"change%1:19:00::","synset_id":"change.n.04"},{"index":3,"sense_key":"change%1:24:00::","synset_id":"change.n.02"},{"index":4,"sense_key":"change%1:06:01::","synset_id":"change.n.06"},{"index":5,"sense_key":"change%1:11:00::","synset_id":"change.n.01"},{"index":6,"sense_key":"change%1:21:03::","synset_id":"change.n.09"},{"index":7,"sense_key":"change%1:21:02::","synset_id":"change.n.05"},{"index":8,"sense_key":"change%1:07:00::","synset_id":"variety.n.06"},{"index":9,"sense_key":"change%1:04:00::","synset_id":"change.n.03"},{"index":10,"sense_key":"change%1:06:00::","synset_id":"change.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"change%1:11:00::","call_ids":["semeval2013.d005.s028.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"change%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":740,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019540000000000002,"input_uncached_usd":0.0037,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.579638000112027}
{"item_id":"semeval2013.d005.s028.t004","gold_sense_keys":["adaptation%1:22:00::"],"candidates":[{"index":1,"sense_key":"adaptation%1:10:00::","synset_id":"adaptation.n.01"},{"index":2,"sense_key":"adaptation%1:04:01::","synset_id":"adaptation.n.03"},{"index":3,"sense_key":"adaptation%1:22:00::","synset_id":"adaptation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"adaptation%1:22:00::","call_ids":["semeval2013.d005.s028.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"adaptation%1:22:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.0037050000000000004,"input_uncached_usd":0.0022050000000000004,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5312240000348538}
{"item_id":"semeval2013.d005.s028.t005","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":2,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":3,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":4,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":5,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":6,"sense_key":"court%1:04:00::","synset_id":"court.n.11"},{"index":7,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":8,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":9,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":10,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":11,"sense_key":"court%1:06:01::","synset_id":"court.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"court%1:14:00::","call_ids":["semeval2013.d005.s028.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":746,"cached_input_tokens":0,"output_tokens":3118,"reasoning_output_tokens":3070},"cost":{"currency":"USD","total_usd":0.09727,"input_uncached_usd":0.0037300000000000002,"input_cached_usd":0.0,"output_usd":0.09354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":58.18818420905154}
{"item_id":"semeval2013.d005.s029.t001","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":3,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":4,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"},{"index":5,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":6,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"claim%1:10:02::","call_ids":["semeval2013.d005.s029.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"claim%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":146},"cost":{"currency":"USD","total_usd":0.007645000000000001,"input_uncached_usd":0.002785,"input_cached_usd":0.0,"output_usd":0.00486,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.804472708026879}
{"item_id":"semeval2013.d005.s030.t000","gold_sense_keys":["filing%1:10:00::"],"candidates":[{"index":1,"sense_key":"filing%1:04:00::","synset_id":"filing.n.04"},{"index":2,"sense_key":"filing%1:04:01::","synset_id":"filing.n.03"},{"index":3,"sense_key":"filing%1:10:00::","synset_id":"filing.n.01"},{"index":4,"sense_key":"filing%1:17:00::","synset_id":"filing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"filing%1:10:00::","call_ids":["semeval2013.d005.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"filing%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":97},"cost":{"currency":"USD","total_usd":0.005725000000000001,"input_uncached_usd":0.0023350000000000003,"input_cached_usd":0.0,"output_usd":0.0033900000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2894956670934334}
{"item_id":"semeval2013.d005.s030.t001","gold_sense_keys":["supreme_court%1:14:00::"],"candidates":[{"index":1,"sense_key":"supreme_court%1:14:01::","synset_id":"supreme_court.n.02"},{"index":2,"sense_key":"supreme_court%1:14:00::","synset_id":"supreme_court.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"supreme_court%1:14:00::","call_ids":["semeval2013.d005.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"supreme_court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.00469,"input_uncached_usd":0.00214,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6879161250544712}
{"item_id":"semeval2013.d005.s030.t002","gold_sense_keys":["decision%1:09:00::"],"candidates":[{"index":1,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"},{"index":2,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"},{"index":3,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"},{"index":4,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"},{"index":5,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"decision%1:09:00::","call_ids":["semeval2013.d005.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"decision%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.00579,"input_uncached_usd":0.0027300000000000002,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6993309590034187}
{"item_id":"semeval2013.d005.s030.t003","gold_sense_keys":["conviction%1:04:00::"],"candidates":[{"index":1,"sense_key":"conviction%1:04:00::","synset_id":"conviction.n.02"},{"index":2,"sense_key":"conviction%1:09:00::","synset_id":"conviction.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conviction%1:04:00::","call_ids":["semeval2013.d005.s030.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conviction%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":425,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004135,"input_uncached_usd":0.002125,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8407320840051398}
{"item_id":"semeval2013.d005.s030.t004","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":3,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"},{"index":4,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":5,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":6,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s030.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":113},"cost":{"currency":"USD","total_usd":0.006985,"input_uncached_usd":0.003115,"input_cached_usd":0.0,"output_usd":0.00387,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.29464574996382}
{"item_id":"semeval2013.d005.s030.t005","gold_sense_keys":["coverage%1:10:00::"],"candidates":[{"index":1,"sense_key":"coverage%1:21:00::","synset_id":"coverage.n.01"},{"index":2,"sense_key":"coverage%1:07:00::","synset_id":"coverage.n.02"},{"index":3,"sense_key":"coverage%1:10:00::","synset_id":"coverage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"coverage%1:10:00::","call_ids":["semeval2013.d005.s030.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"coverage%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.0037400000000000003,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.445317417033948}
{"item_id":"semeval2013.d005.s030.t006","gold_sense_keys":["rule%1:10:00::"],"candidates":[{"index":1,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"},{"index":2,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":3,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":4,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":5,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":6,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":7,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":8,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":9,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":10,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":11,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":12,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"rule%1:10:00::","call_ids":["semeval2013.d005.s030.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"rule%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":877,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":147},"cost":{"currency":"USD","total_usd":0.009245,"input_uncached_usd":0.004385,"input_cached_usd":0.0,"output_usd":0.00486,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.125474333995953}
{"item_id":"semeval2013.d005.s031.t000","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":2,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":3,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":4,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":5,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":6,"sense_key":"case%1:06:01::","synset_id":"case.n.20"},{"index":7,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":8,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":9,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":10,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":11,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":12,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":13,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":14,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":15,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":16,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":17,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":18,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":19,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":20,"sense_key":"case%1:10:00::","synset_id":"case.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d005.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1157,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":129},"cost":{"currency":"USD","total_usd":0.010135000000000002,"input_uncached_usd":0.005785,"input_cached_usd":0.0,"output_usd":0.00435,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0585534169804305}
{"item_id":"semeval2013.d005.s031.t001","gold_sense_keys":["century%1:28:00::"],"candidates":[{"index":1,"sense_key":"century%1:28:00::","synset_id":"century.n.01"},{"index":2,"sense_key":"century%1:23:00::","synset_id":"hundred.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"century%1:28:00::","call_ids":["semeval2013.d005.s031.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"century%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":354,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.00339,"input_uncached_usd":0.00177,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.502640499966219}
{"item_id":"semeval2013.d006.s000.t001","gold_sense_keys":["fraud%1:04:01::"],"candidates":[{"index":1,"sense_key":"fraud%1:18:00::","synset_id":"imposter.n.01"},{"index":2,"sense_key":"fraud%1:04:01::","synset_id":"fraud.n.03"},{"index":3,"sense_key":"fraud%1:04:00::","synset_id":"fraud.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fraud%1:04:00::","call_ids":["semeval2013.d006.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fraud%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":298,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01733,"input_uncached_usd":0.0014900000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.32483916694764}
{"item_id":"semeval2013.d006.s000.t002","gold_sense_keys":["law%1:14:00::"],"candidates":[{"index":1,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":2,"sense_key":"law%1:14:01::","synset_id":"police.n.01"},{"index":3,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":4,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":5,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":6,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":7,"sense_key":"law%1:09:01::","synset_id":"law.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"law%1:14:00::","call_ids":["semeval2013.d006.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"law%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01832,"input_uncached_usd":0.00248,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.51203758304473}
{"item_id":"semeval2013.d006.s001.t000","gold_sense_keys":["trick%1:04:05::"],"candidates":[{"index":1,"sense_key":"trick%1:28:00::","synset_id":"trick.n.02"},{"index":2,"sense_key":"trick%1:04:01::","synset_id":"trick.n.07"},{"index":3,"sense_key":"trick%1:04:04::","synset_id":"trick.n.03"},{"index":4,"sense_key":"trick%1:04:02::","synset_id":"magic_trick.n.01"},{"index":5,"sense_key":"trick%1:18:00::","synset_id":"whoremaster.n.01"},{"index":6,"sense_key":"trick%1:04:05::","synset_id":"trick.n.01"},{"index":7,"sense_key":"trick%1:04:00::","synset_id":"antic.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"trick%1:04:05::","call_ids":["semeval2013.d006.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"trick%1:04:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":246,"reasoning_output_tokens":230},"cost":{"currency":"USD","total_usd":0.010020000000000001,"input_uncached_usd":0.0026400000000000004,"input_cached_usd":0.0,"output_usd":0.00738,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.153253708034754}
{"item_id":"semeval2013.d006.s001.t002","gold_sense_keys":["will%1:09:01::"],"candidates":[{"index":1,"sense_key":"will%1:09:00::","synset_id":"volition.n.01"},{"index":2,"sense_key":"will%1:09:01::","synset_id":"will.n.02"},{"index":3,"sense_key":"will%1:10:00::","synset_id":"will.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"will%1:09:01::","call_ids":["semeval2013.d006.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"will%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":363,"cached_input_tokens":0,"output_tokens":276,"reasoning_output_tokens":261},"cost":{"currency":"USD","total_usd":0.010095000000000002,"input_uncached_usd":0.0018150000000000002,"input_cached_usd":0.0,"output_usd":0.008280000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.723778665997088}
{"item_id":"semeval2013.d006.s002.t000","gold_sense_keys":["expulsion%1:04:01::"],"candidates":[{"index":1,"sense_key":"expulsion%1:04:00::","synset_id":"expulsion.n.03"},{"index":2,"sense_key":"expulsion%1:04:01::","synset_id":"ejection.n.02"},{"index":3,"sense_key":"expulsion%1:04:02::","synset_id":"extrusion.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"expulsion%1:04:01::","call_ids":["semeval2013.d006.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"expulsion%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":107},"cost":{"currency":"USD","total_usd":0.005855000000000001,"input_uncached_usd":0.0021650000000000003,"input_cached_usd":0.0,"output_usd":0.00369,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.012268375023268}
{"item_id":"semeval2013.d006.s002.t001","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":2,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":3,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":4,"sense_key":"player%1:18:03::","synset_id":"player.n.05"},{"index":5,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":40,"reasoning_output_tokens":25},"cost":{"currency":"USD","total_usd":0.0034850000000000003,"input_uncached_usd":0.002285,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.382704084040597}
{"item_id":"semeval2013.d006.s002.t002","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":2,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":3,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":4,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":5,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":6,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":7,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":8,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":9,"sense_key":"game%1:04:04::","synset_id":"game.n.11"},{"index":10,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":11,"sense_key":"game%1:04:02::","synset_id":"game.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d006.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.006280000000000001,"input_uncached_usd":0.00355,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.032819041982293}
{"item_id":"semeval2013.d006.s002.t003","gold_sense_keys":["ink%1:27:01::"],"candidates":[{"index":1,"sense_key":"ink%1:27:01::","synset_id":"ink.n.01"},{"index":2,"sense_key":"ink%1:08:00::","synset_id":"ink.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ink%1:27:01::","call_ids":["semeval2013.d006.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ink%1:27:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":346,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.00395,"input_uncached_usd":0.0017300000000000002,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7918595420196652}
{"item_id":"semeval2013.d006.s002.t004","gold_sense_keys":["behavior%1:04:00::"],"candidates":[{"index":1,"sense_key":"behavior%1:04:01::","synset_id":"behavior.n.04"},{"index":2,"sense_key":"behavior%1:07:00::","synset_id":"demeanor.n.01"},{"index":3,"sense_key":"behavior%1:26:00::","synset_id":"behavior.n.02"},{"index":4,"sense_key":"behavior%1:04:00::","synset_id":"behavior.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"behavior%1:04:00::","call_ids":["semeval2013.d006.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"behavior%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":161,"reasoning_output_tokens":146},"cost":{"currency":"USD","total_usd":0.007105,"input_uncached_usd":0.002275,"input_cached_usd":0.0,"output_usd":0.00483,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7210047079715878}
{"item_id":"semeval2013.d006.s002.t005","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:03::","synset_id":"player.n.05"},{"index":5,"sense_key":"player%1:18:04::","synset_id":"player.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.003725,"input_uncached_usd":0.002285,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5483385829720646}
{"item_id":"semeval2013.d006.s003.t000","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":2,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":3,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":4,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":5,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":6,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"minute%1:28:00::","call_ids":["semeval2013.d006.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.00493,"input_uncached_usd":0.00289,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.071392207988538}
{"item_id":"semeval2013.d006.s003.t001","gold_sense_keys":["end%1:09:00::","end%1:11:00::","end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:15:01::","synset_id":"end.n.09"},{"index":2,"sense_key":"end%1:04:02::","synset_id":"end.n.14"},{"index":3,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":4,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":5,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":6,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":7,"sense_key":"end%1:15:00::","synset_id":"end.n.01"},{"index":8,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"},{"index":9,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":10,"sense_key":"end%1:18:00::","synset_id":"end.n.08"},{"index":11,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":12,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":13,"sense_key":"end%1:15:02::","synset_id":"end.n.07"},{"index":14,"sense_key":"end%1:11:00::","synset_id":"end.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"end%1:28:00::","call_ids":["semeval2013.d006.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"end%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":928,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":147},"cost":{"currency":"USD","total_usd":0.0095,"input_uncached_usd":0.00464,"input_cached_usd":0.0,"output_usd":0.00486,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.535528082982637}
{"item_id":"semeval2013.d006.s003.t002","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":2,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":3,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":6,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":7,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":8,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":9,"sense_key":"game%1:04:04::","synset_id":"game.n.11"},{"index":10,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":11,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d006.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":206,"reasoning_output_tokens":190},"cost":{"currency":"USD","total_usd":0.009970000000000001,"input_uncached_usd":0.0037900000000000004,"input_cached_usd":0.0,"output_usd":0.00618,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5888433329528198}
{"item_id":"semeval2013.d006.s003.t003","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:01::","synset_id":"team.n.02"},{"index":2,"sense_key":"team%1:14:00::","synset_id":"team.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":29,"reasoning_output_tokens":14},"cost":{"currency":"USD","total_usd":0.00282,"input_uncached_usd":0.0019500000000000001,"input_cached_usd":0.0,"output_usd":0.00087,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.1382435840787366}
{"item_id":"semeval2013.d006.s003.t004","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":2,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":3,"sense_key":"player%1:18:03::","synset_id":"player.n.05"},{"index":4,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":5,"sense_key":"player%1:18:01::","synset_id":"player.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004295,"input_uncached_usd":0.0025250000000000003,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9426370420260355}
{"item_id":"semeval2013.d006.s003.t005","gold_sense_keys":["time%1:28:05::"],"candidates":[{"index":1,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":2,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"},{"index":3,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":4,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":7,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":8,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":9,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":10,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"time%1:28:05::","call_ids":["semeval2013.d006.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"time%1:28:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":739,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":134},"cost":{"currency":"USD","total_usd":0.008195000000000001,"input_uncached_usd":0.0036950000000000004,"input_cached_usd":0.0,"output_usd":0.0045000000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5960969580337405}
{"item_id":"semeval2013.d006.s003.t006","gold_sense_keys":["referee%1:18:00::"],"candidates":[{"index":1,"sense_key":"referee%1:18:02::","synset_id":"referee.n.03"},{"index":2,"sense_key":"referee%1:18:00::","synset_id":"referee.n.01"},{"index":3,"sense_key":"referee%1:18:01::","synset_id":"reviewer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"referee%1:18:00::","call_ids":["semeval2013.d006.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"referee%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.0037750000000000006,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8321630000136793}
{"item_id":"semeval2013.d006.s004.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":2,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":3,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":4,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":5,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":6,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":7,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":8,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":9,"sense_key":"way%1:21:00::","synset_id":"way.n.12"},{"index":10,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":11,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":12,"sense_key":"way%1:04:01::","synset_id":"means.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"way%1:04:01::","call_ids":["semeval2013.d006.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":836,"cached_input_tokens":0,"output_tokens":222,"reasoning_output_tokens":206},"cost":{"currency":"USD","total_usd":0.01084,"input_uncached_usd":0.0041800000000000006,"input_cached_usd":0.0,"output_usd":0.00666,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7045288330409676}
{"item_id":"semeval2013.d006.s004.t002","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:09:00::","synset_id":"group.n.03"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:03:00::","synset_id":"group.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"group%1:03:00::","call_ids":["semeval2013.d006.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.004275,"input_uncached_usd":0.002415,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7672442919574678}
{"item_id":"semeval2013.d006.s004.t003","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":2,"sense_key":"player%1:18:03::","synset_id":"player.n.05"},{"index":3,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":4,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":5,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.004215,"input_uncached_usd":0.0026850000000000003,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.071190042071976}
{"item_id":"semeval2013.d006.s004.t005","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":2,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":9,"sense_key":"game%1:04:04::","synset_id":"game.n.11"},{"index":10,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":11,"sense_key":"game%1:04:02::","synset_id":"game.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d006.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":790,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.00668,"input_uncached_usd":0.00395,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.769878208055161}
{"item_id":"semeval2013.d006.s004.t006","gold_sense_keys":["league%1:14:01::"],"candidates":[{"index":1,"sense_key":"league%1:14:00::","synset_id":"league.n.02"},{"index":2,"sense_key":"league%1:14:01::","synset_id":"league.n.01"},{"index":3,"sense_key":"league%1:23:00::","synset_id":"league.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"league%1:14:01::","call_ids":["semeval2013.d006.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"league%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":88},"cost":{"currency":"USD","total_usd":0.0053950000000000005,"input_uncached_usd":0.002275,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.49864908400923}
{"item_id":"semeval2013.d006.s004.t008","gold_sense_keys":["phase%1:28:00::"],"candidates":[{"index":1,"sense_key":"phase%1:07:00::","synset_id":"phase.n.04"},{"index":2,"sense_key":"phase%1:26:00::","synset_id":"phase.n.02"},{"index":3,"sense_key":"phase%1:28:00::","synset_id":"phase.n.01"},{"index":4,"sense_key":"phase%1:28:01::","synset_id":"phase.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"phase%1:28:00::","call_ids":["semeval2013.d006.s004.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"phase%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.004985,"input_uncached_usd":0.0028850000000000004,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1276016250485554}
{"item_id":"semeval2013.d006.s005.t000","gold_sense_keys":["committee%1:14:00::"],"candidates":[{"index":1,"sense_key":"committee%1:14:01::","synset_id":"committee.n.02"},{"index":2,"sense_key":"committee%1:14:00::","synset_id":"committee.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"committee%1:14:00::","call_ids":["semeval2013.d006.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"committee%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.00476,"input_uncached_usd":0.00236,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9093500419985503}
{"item_id":"semeval2013.d006.s005.t002","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":5,"sense_key":"game%1:04:04::","synset_id":"game.n.11"},{"index":6,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":7,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":8,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":9,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":10,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":11,"sense_key":"game%1:28:00::","synset_id":"game.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d006.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":818,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":81},"cost":{"currency":"USD","total_usd":0.007000000000000001,"input_uncached_usd":0.004090000000000001,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2673043749528006}
{"item_id":"semeval2013.d006.s005.t003","gold_sense_keys":["trainer%1:18:00::"],"candidates":[{"index":1,"sense_key":"trainer%1:18:00::","synset_id":"trainer.n.01"},{"index":2,"sense_key":"trainer%1:06:00::","synset_id":"flight_simulator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trainer%1:18:00::","call_ids":["semeval2013.d006.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trainer%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":458,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.004,"input_uncached_usd":0.0022900000000000004,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8251664170529693}
{"item_id":"semeval2013.d006.s005.t004","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.004105,"input_uncached_usd":0.002245,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.792367040994577}
{"item_id":"semeval2013.d006.s005.t005","gold_sense_keys":["amount%1:21:00::"],"candidates":[{"index":1,"sense_key":"amount%1:07:00::","synset_id":"amount.n.02"},{"index":2,"sense_key":"amount%1:03:00::","synset_id":"measure.n.02"},{"index":3,"sense_key":"amount%1:09:00::","synset_id":"sum.n.02"},{"index":4,"sense_key":"amount%1:21:00::","synset_id":"sum.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"amount%1:21:00::","call_ids":["semeval2013.d006.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"amount%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.004675,"input_uncached_usd":0.002755,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.419952458003536}
{"item_id":"semeval2013.d006.s005.t006","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":2,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:03::","synset_id":"player.n.05"},{"index":5,"sense_key":"player%1:18:04::","synset_id":"player.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s005.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":564,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00477,"input_uncached_usd":0.00282,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7119654170237482}
{"item_id":"semeval2013.d006.s006.t000","gold_sense_keys":["provocation%1:16:00::"],"candidates":[{"index":1,"sense_key":"provocation%1:16:00::","synset_id":"incitement.n.03"},{"index":2,"sense_key":"provocation%1:10:00::","synset_id":"provocation.n.03"},{"index":3,"sense_key":"provocation%1:04:00::","synset_id":"aggravation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"provocation%1:16:00::","call_ids":["semeval2013.d006.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"provocation%1:16:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":324,"reasoning_output_tokens":308},"cost":{"currency":"USD","total_usd":0.012385,"input_uncached_usd":0.0026650000000000003,"input_cached_usd":0.0,"output_usd":0.00972,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.167273749946617}
{"item_id":"semeval2013.d006.s006.t002","gold_sense_keys":["circumstance%1:26:01::","circumstance%1:26:02::"],"candidates":[{"index":1,"sense_key":"circumstance%1:09:00::","synset_id":"circumstance.n.03"},{"index":2,"sense_key":"circumstance%1:11:00::","synset_id":"circumstance.n.04"},{"index":3,"sense_key":"circumstance%1:26:02::","synset_id":"context.n.02"},{"index":4,"sense_key":"circumstance%1:26:01::","synset_id":"circumstance.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"circumstance%1:26:02::","call_ids":["semeval2013.d006.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"circumstance%1:26:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":564,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01866,"input_uncached_usd":0.00282,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.070298749953508}
{"item_id":"semeval2013.d006.s006.t003","gold_sense_keys":["practice%1:04:00::"],"candidates":[{"index":1,"sense_key":"practice%1:04:04::","synset_id":"practice.n.03"},{"index":2,"sense_key":"practice%1:09:00::","synset_id":"practice.n.05"},{"index":3,"sense_key":"practice%1:04:00::","synset_id":"practice.n.01"},{"index":4,"sense_key":"practice%1:04:02::","synset_id":"exercise.n.03"},{"index":5,"sense_key":"practice%1:04:03::","synset_id":"practice.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"practice%1:04:00::","call_ids":["semeval2013.d006.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"practice%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":158,"reasoning_output_tokens":143},"cost":{"currency":"USD","total_usd":0.007855000000000001,"input_uncached_usd":0.003115,"input_cached_usd":0.0,"output_usd":0.00474,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6286954999668524}
{"item_id":"semeval2013.d006.s006.t004","gold_sense_keys":["world%1:14:01::"],"candidates":[{"index":1,"sense_key":"world%1:17:00::","synset_id":"earth.n.01"},{"index":2,"sense_key":"world%1:14:01::","synset_id":"world.n.02"},{"index":3,"sense_key":"world%1:09:01::","synset_id":"worldly_concern.n.01"},{"index":4,"sense_key":"world%1:14:02::","synset_id":"populace.n.01"},{"index":5,"sense_key":"world%1:17:02::","synset_id":"world.n.06"},{"index":6,"sense_key":"world%1:09:00::","synset_id":"world.n.03"},{"index":7,"sense_key":"world%1:05:00::","synset_id":"world.n.08"},{"index":8,"sense_key":"world%1:17:01::","synset_id":"universe.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"world%1:14:01::","call_ids":["semeval2013.d006.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"world%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":693,"reasoning_output_tokens":671},"cost":{"currency":"USD","total_usd":0.024845000000000002,"input_uncached_usd":0.0040550000000000004,"input_cached_usd":0.0,"output_usd":0.02079,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.112094083917327}
{"item_id":"semeval2013.d006.s006.t005","gold_sense_keys":["football%1:04:00::"],"candidates":[{"index":1,"sense_key":"football%1:06:00::","synset_id":"football.n.02"},{"index":2,"sense_key":"football%1:04:00::","synset_id":"football.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"football%1:04:00::","call_ids":["semeval2013.d006.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"football%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.004145,"input_uncached_usd":0.002465,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7067607910139486}
{"item_id":"semeval2013.d006.s007.t001","gold_sense_keys":["rule%1:10:02::"],"candidates":[{"index":1,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":2,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":3,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":4,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":5,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":6,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"},{"index":7,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":8,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":9,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":10,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":11,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":12,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"rule%1:09:00::","call_ids":["semeval2013.d006.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"rule%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1016,"cached_input_tokens":0,"output_tokens":484,"reasoning_output_tokens":468},"cost":{"currency":"USD","total_usd":0.0196,"input_uncached_usd":0.00508,"input_cached_usd":0.0,"output_usd":0.01452,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.560958457994275}
{"item_id":"semeval2013.d006.s008.t000","gold_sense_keys":["part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":2,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":3,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":6,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":7,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":8,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"},{"index":9,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":10,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":11,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":12,"sense_key":"part%1:10:00::","synset_id":"part.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"part%1:09:00::","call_ids":["semeval2013.d006.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"part%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1052,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0211,"input_uncached_usd":0.005260000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.401493625016883}
{"item_id":"semeval2013.d006.s008.t001","gold_sense_keys":["debate%1:10:01::"],"candidates":[{"index":1,"sense_key":"debate%1:10:00::","synset_id":"debate.n.02"},{"index":2,"sense_key":"debate%1:10:01::","synset_id":"argument.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"debate%1:10:01::","call_ids":["semeval2013.d006.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"debate%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.004905,"input_uncached_usd":0.002775,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.254221500013955}
{"item_id":"semeval2013.d006.s008.t002","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":2,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":3,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":4,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":5,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":6,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":7,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":8,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":9,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":10,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":11,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":12,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":13,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":14,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":15,"sense_key":"case%1:06:01::","synset_id":"case.n.20"},{"index":16,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":17,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":18,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":19,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":20,"sense_key":"case%1:06:02::","synset_id":"case.n.19"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"case%1:26:00::","call_ids":["semeval2013.d006.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"case%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1321,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.022445,"input_uncached_usd":0.006605000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.066356750088744}
{"item_id":"semeval2013.d006.s008.t003","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"},{"index":2,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":3,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":4,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fact%1:26:01::","call_ids":["semeval2013.d006.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fact%1:26:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":2082,"reasoning_output_tokens":2049},"cost":{"currency":"USD","total_usd":0.065625,"input_uncached_usd":0.0031650000000000003,"input_cached_usd":0.0,"output_usd":0.06246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":31.263454083004035}
{"item_id":"semeval2013.d006.s008.t004","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:01::","synset_id":"team.n.02"},{"index":2,"sense_key":"team%1:14:00::","synset_id":"team.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.0045850000000000005,"input_uncached_usd":0.002575,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4939610420260578}
{"item_id":"semeval2013.d006.s008.t005","gold_sense_keys":["organization%1:14:00::","organization%1:14:01::"],"candidates":[{"index":1,"sense_key":"organization%1:04:01::","synset_id":"organization.n.06"},{"index":2,"sense_key":"organization%1:04:00::","synset_id":"organization.n.04"},{"index":3,"sense_key":"organization%1:07:00::","synset_id":"organization.n.05"},{"index":4,"sense_key":"organization%1:09:00::","synset_id":"arrangement.n.03"},{"index":5,"sense_key":"organization%1:14:00::","synset_id":"organization.n.01"},{"index":6,"sense_key":"organization%1:14:01::","synset_id":"administration.n.02"},{"index":7,"sense_key":"organization%1:04:02::","synset_id":"constitution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"organization%1:14:00::","call_ids":["semeval2013.d006.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"organization%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":835,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":129},"cost":{"currency":"USD","total_usd":0.008525000000000001,"input_uncached_usd":0.004175000000000001,"input_cached_usd":0.0,"output_usd":0.00435,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.243792916997336}
{"item_id":"semeval2013.d006.s008.t007","gold_sense_keys":["nobility%1:07:00::"],"candidates":[{"index":1,"sense_key":"nobility%1:26:00::","synset_id":"nobility.n.03"},{"index":2,"sense_key":"nobility%1:07:00::","synset_id":"nobility.n.02"},{"index":3,"sense_key":"nobility%1:14:00::","synset_id":"nobility.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"nobility%1:07:00::","call_ids":["semeval2013.d006.s008.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"nobility%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004845,"input_uncached_usd":0.0028350000000000003,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.926436125067994}
{"item_id":"semeval2013.d006.s008.t008","gold_sense_keys":["honor%1:07:00::"],"candidates":[{"index":1,"sense_key":"honor%1:26:00::","synset_id":"honor.n.02"},{"index":2,"sense_key":"honor%1:10:00::","synset_id":"award.n.02"},{"index":3,"sense_key":"honor%1:07:00::","synset_id":"honor.n.03"},{"index":4,"sense_key":"honor%1:07:01::","synset_id":"honor.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"honor%1:07:00::","call_ids":["semeval2013.d006.s008.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"honor%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.00489,"input_uncached_usd":0.00303,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8858347079949453}
{"item_id":"semeval2013.d006.s008.t009","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:03::","synset_id":"player.n.05"},{"index":4,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":5,"sense_key":"player%1:18:04::","synset_id":"player.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s008.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.00474,"input_uncached_usd":0.0031500000000000005,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.786794374929741}
{"item_id":"semeval2013.d006.s008.t010","gold_sense_keys":["act%1:03:00::"],"candidates":[{"index":1,"sense_key":"act%1:03:00::","synset_id":"act.n.02"},{"index":2,"sense_key":"act%1:10:01::","synset_id":"act.n.01"},{"index":3,"sense_key":"act%1:10:00::","synset_id":"act.n.03"},{"index":4,"sense_key":"act%1:10:03::","synset_id":"act.n.05"},{"index":5,"sense_key":"act%1:10:02::","synset_id":"act.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"act%1:03:00::","call_ids":["semeval2013.d006.s008.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"act%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.0048850000000000005,"input_uncached_usd":0.0032350000000000005,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7765235420083627}
{"item_id":"semeval2013.d006.s008.t012","gold_sense_keys":["history%1:28:00::"],"candidates":[{"index":1,"sense_key":"history%1:09:01::","synset_id":"history.n.05"},{"index":2,"sense_key":"history%1:10:00::","synset_id":"history.n.02"},{"index":3,"sense_key":"history%1:28:02::","synset_id":"history.n.04"},{"index":4,"sense_key":"history%1:28:00::","synset_id":"history.n.01"},{"index":5,"sense_key":"history%1:09:00::","synset_id":"history.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"history%1:28:00::","call_ids":["semeval2013.d006.s008.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"history%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":674,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01921,"input_uncached_usd":0.00337,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.678023457992822}
{"item_id":"semeval2013.d006.s009.t000","gold_sense_keys":["debate%1:10:01::"],"candidates":[{"index":1,"sense_key":"debate%1:10:00::","synset_id":"debate.n.02"},{"index":2,"sense_key":"debate%1:10:01::","synset_id":"argument.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"debate%1:10:01::","call_ids":["semeval2013.d006.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"debate%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.004625000000000001,"input_uncached_usd":0.002645,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7418135419720784}
{"item_id":"semeval2013.d006.s009.t001","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:06:00::","synset_id":"type.n.06"},{"index":2,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":3,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":4,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":5,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":6,"sense_key":"type%1:10:00::","synset_id":"type.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2013.d006.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":670,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.00476,"input_uncached_usd":0.00335,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6967472080141306}
{"item_id":"semeval2013.d006.s009.t002","gold_sense_keys":["behavior%1:04:00::"],"candidates":[{"index":1,"sense_key":"behavior%1:26:00::","synset_id":"behavior.n.02"},{"index":2,"sense_key":"behavior%1:04:01::","synset_id":"behavior.n.04"},{"index":3,"sense_key":"behavior%1:07:00::","synset_id":"demeanor.n.01"},{"index":4,"sense_key":"behavior%1:04:00::","synset_id":"behavior.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"behavior%1:04:00::","call_ids":["semeval2013.d006.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"behavior%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":112},"cost":{"currency":"USD","total_usd":0.00682,"input_uncached_usd":0.00301,"input_cached_usd":0.0,"output_usd":0.00381,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8409026250010356}
{"item_id":"semeval2013.d006.s009.t003","gold_sense_keys":["principle%1:09:01::"],"candidates":[{"index":1,"sense_key":"principle%1:09:00::","synset_id":"principle.n.03"},{"index":2,"sense_key":"principle%1:10:00::","synset_id":"principle.n.05"},{"index":3,"sense_key":"principle%1:09:02::","synset_id":"rationale.n.01"},{"index":4,"sense_key":"principle%1:09:03::","synset_id":"principle.n.01"},{"index":5,"sense_key":"principle%1:09:01::","synset_id":"principle.n.02"},{"index":6,"sense_key":"principle%1:09:04::","synset_id":"principle.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"principle%1:09:01::","call_ids":["semeval2013.d006.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"principle%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":700,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01934,"input_uncached_usd":0.0035,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.067368166986853}
{"item_id":"semeval2013.d006.s009.t005","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:04:00::","synset_id":"action.n.10"},{"index":3,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":4,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":5,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":6,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":7,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":8,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":9,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":10,"sense_key":"action%1:04:04::","synset_id":"action.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":890,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":98},"cost":{"currency":"USD","total_usd":0.00784,"input_uncached_usd":0.00445,"input_cached_usd":0.0,"output_usd":0.0033900000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3580714999698102}
{"item_id":"semeval2013.d006.s009.t006","gold_sense_keys":["fraud%1:04:01::"],"candidates":[{"index":1,"sense_key":"fraud%1:04:00::","synset_id":"fraud.n.01"},{"index":2,"sense_key":"fraud%1:18:00::","synset_id":"imposter.n.01"},{"index":3,"sense_key":"fraud%1:04:01::","synset_id":"fraud.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fraud%1:04:00::","call_ids":["semeval2013.d006.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fraud%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018605,"input_uncached_usd":0.002765,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":17.570360166952014}
{"item_id":"semeval2013.d006.s009.t007","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":2,"sense_key":"action%1:04:00::","synset_id":"action.n.10"},{"index":3,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":4,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":5,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":6,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":7,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":8,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":9,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":10,"sense_key":"action%1:06:00::","synset_id":"action.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":890,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.00676,"input_uncached_usd":0.00445,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.015256750048138}
{"item_id":"semeval2013.d006.s009.t009","gold_sense_keys":["regulation%1:10:00::"],"candidates":[{"index":1,"sense_key":"regulation%1:04:01::","synset_id":"regulation.n.05"},{"index":2,"sense_key":"regulation%1:26:00::","synset_id":"regulation.n.03"},{"index":3,"sense_key":"regulation%1:10:00::","synset_id":"regulation.n.01"},{"index":4,"sense_key":"regulation%1:22:00::","synset_id":"regulation.n.04"},{"index":5,"sense_key":"regulation%1:09:00::","synset_id":"rule.n.01"},{"index":6,"sense_key":"regulation%1:04:00::","synset_id":"regulation.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"regulation%1:10:00::","call_ids":["semeval2013.d006.s009.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"regulation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":85},"cost":{"currency":"USD","total_usd":0.0063,"input_uncached_usd":0.0032700000000000003,"input_cached_usd":0.0,"output_usd":0.00303,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3713114999700338}
{"item_id":"semeval2013.d006.s010.t000","gold_sense_keys":["matter%1:09:00::"],"candidates":[{"index":1,"sense_key":"matter%1:09:00::","synset_id":"topic.n.02"},{"index":2,"sense_key":"matter%1:07:00::","synset_id":"matter.n.05"},{"index":3,"sense_key":"matter%1:10:00::","synset_id":"matter.n.06"},{"index":4,"sense_key":"matter%1:09:01::","synset_id":"matter.n.01"},{"index":5,"sense_key":"matter%1:09:02::","synset_id":"matter.n.04"},{"index":6,"sense_key":"matter%1:03:00::","synset_id":"matter.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"matter%1:09:00::","call_ids":["semeval2013.d006.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"matter%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":287,"reasoning_output_tokens":271},"cost":{"currency":"USD","total_usd":0.011935,"input_uncached_usd":0.0033250000000000003,"input_cached_usd":0.0,"output_usd":0.00861,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.4425523749087}
{"item_id":"semeval2013.d006.s010.t001","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":2,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":3,"sense_key":"type%1:06:00::","synset_id":"type.n.06"},{"index":4,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":5,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":6,"sense_key":"type%1:10:00::","synset_id":"type.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2013.d006.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.005255,"input_uncached_usd":0.003305,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8637538329930976}
{"item_id":"semeval2013.d006.s010.t002","gold_sense_keys":["damage%1:11:00::"],"candidates":[{"index":1,"sense_key":"damage%1:04:01::","synset_id":"wrong.n.02"},{"index":2,"sense_key":"damage%1:11:00::","synset_id":"damage.n.01"},{"index":3,"sense_key":"damage%1:21:00::","synset_id":"price.n.02"},{"index":4,"sense_key":"damage%1:04:00::","synset_id":"damage.n.03"},{"index":5,"sense_key":"damage%1:11:01::","synset_id":"damage.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"damage%1:11:00::","call_ids":["semeval2013.d006.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"damage%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":234,"reasoning_output_tokens":218},"cost":{"currency":"USD","total_usd":0.010005,"input_uncached_usd":0.0029850000000000002,"input_cached_usd":0.0,"output_usd":0.00702,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.997297374997288}
{"item_id":"semeval2013.d006.s011.t000","gold_sense_keys":["advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"},{"index":2,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"},{"index":3,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"advantage%1:07:01::","call_ids":["semeval2013.d006.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"advantage%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":268,"reasoning_output_tokens":252},"cost":{"currency":"USD","total_usd":0.010785,"input_uncached_usd":0.002745,"input_cached_usd":0.0,"output_usd":0.00804,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.87447429203894}
{"item_id":"semeval2013.d006.s011.t003","gold_sense_keys":["integrity%1:26:00::"],"candidates":[{"index":1,"sense_key":"integrity%1:07:00::","synset_id":"integrity.n.02"},{"index":2,"sense_key":"integrity%1:26:00::","synset_id":"integrity.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"integrity%1:26:00::","call_ids":["semeval2013.d006.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"integrity%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.0049250000000000006,"input_uncached_usd":0.002735,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9703857919666916}
{"item_id":"semeval2013.d006.s011.t004","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":2,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":3,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":4,"sense_key":"player%1:18:03::","synset_id":"player.n.05"},{"index":5,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":603,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.004545,"input_uncached_usd":0.0030150000000000003,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4814371659886092}
{"item_id":"semeval2013.d006.s011.t005","gold_sense_keys":["aspect%1:07:02::"],"candidates":[{"index":1,"sense_key":"aspect%1:09:00::","synset_id":"aspect.n.01"},{"index":2,"sense_key":"aspect%1:07:02::","synset_id":"aspect.n.02"},{"index":3,"sense_key":"aspect%1:24:00::","synset_id":"aspect.n.04"},{"index":4,"sense_key":"aspect%1:09:01::","synset_id":"view.n.02"},{"index":5,"sense_key":"aspect%1:07:01::","synset_id":"expression.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"aspect%1:09:00::","call_ids":["semeval2013.d006.s011.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"aspect%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":934,"reasoning_output_tokens":913},"cost":{"currency":"USD","total_usd":0.0311,"input_uncached_usd":0.0030800000000000003,"input_cached_usd":0.0,"output_usd":0.02802,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":29.428701041964814}
{"item_id":"semeval2013.d006.s011.t006","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":5,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":6,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":7,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":8,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":9,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":890,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.00739,"input_uncached_usd":0.00445,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5708842090098187}
{"item_id":"semeval2013.d006.s011.t007","gold_sense_keys":["danger%1:26:00::"],"candidates":[{"index":1,"sense_key":"danger%1:26:00::","synset_id":"danger.n.01"},{"index":2,"sense_key":"danger%1:04:00::","synset_id":"risk.n.02"},{"index":3,"sense_key":"danger%1:15:00::","synset_id":"danger.n.04"},{"index":4,"sense_key":"danger%1:26:01::","synset_id":"danger.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"danger%1:26:00::","call_ids":["semeval2013.d006.s011.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"danger%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":93},"cost":{"currency":"USD","total_usd":0.006195,"input_uncached_usd":0.002955,"input_cached_usd":0.0,"output_usd":0.0032400000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9069496659794822}
{"item_id":"semeval2013.d006.s012.t000","gold_sense_keys":["advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"},{"index":2,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"},{"index":3,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"advantage%1:07:01::","call_ids":["semeval2013.d006.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"advantage%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01876,"input_uncached_usd":0.0029200000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.248433125088923}
{"item_id":"semeval2013.d006.s012.t001","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:01::","synset_id":"team.n.02"},{"index":2,"sense_key":"team%1:14:00::","synset_id":"team.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.004660000000000001,"input_uncached_usd":0.0026200000000000004,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.635020334040746}
{"item_id":"semeval2013.d006.s012.t003","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:03::","synset_id":"player.n.05"},{"index":2,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":3,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.005445,"input_uncached_usd":0.0031950000000000004,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.057748791994527}
{"item_id":"semeval2013.d006.s012.t004","gold_sense_keys":["take_advantage%2:40:00::"],"candidates":[{"index":1,"sense_key":"take_advantage%2:41:12::","synset_id":"trespass.v.02"},{"index":2,"sense_key":"take_advantage%2:40:00::","synset_id":"capitalize.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"take_advantage%2:40:00::","call_ids":["semeval2013.d006.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"take_advantage%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":309,"reasoning_output_tokens":293},"cost":{"currency":"USD","total_usd":0.012115,"input_uncached_usd":0.0028450000000000003,"input_cached_usd":0.0,"output_usd":0.00927,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.35326637502294}
{"item_id":"semeval2013.d006.s012.t005","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:01::","synset_id":"team.n.02"},{"index":2,"sense_key":"team%1:14:00::","synset_id":"team.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":523,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.004295,"input_uncached_usd":0.002615,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3694872499909252}
{"item_id":"semeval2013.d006.s013.t000","gold_sense_keys":["thing%1:07:00::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":3,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":4,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":5,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":6,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":7,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":8,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"},{"index":9,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":10,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":11,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":12,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"thing%1:07:00::","call_ids":["semeval2013.d006.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"thing%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":187,"reasoning_output_tokens":171},"cost":{"currency":"USD","total_usd":0.00987,"input_uncached_usd":0.004260000000000001,"input_cached_usd":0.0,"output_usd":0.00561,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7501964999828488}
{"item_id":"semeval2013.d006.s013.t001","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":2,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":3,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":4,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":5,"sense_key":"action%1:04:00::","synset_id":"action.n.10"},{"index":6,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":7,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":8,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":9,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":10,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":912,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":79},"cost":{"currency":"USD","total_usd":0.007380000000000001,"input_uncached_usd":0.004560000000000001,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2631336669437587}
{"item_id":"semeval2013.d006.s013.t002","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":2,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":3,"sense_key":"player%1:18:03::","synset_id":"player.n.05"},{"index":4,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":5,"sense_key":"player%1:18:01::","synset_id":"player.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.00466,"input_uncached_usd":0.0031300000000000004,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.645716957980767}
{"item_id":"semeval2013.d006.s013.t005","gold_sense_keys":["rule%1:10:02::"],"candidates":[{"index":1,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":2,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":3,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":4,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":5,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":6,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":7,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":8,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"},{"index":9,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":10,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":11,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":12,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"rule%1:10:02::","call_ids":["semeval2013.d006.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"rule%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1006,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.007430000000000001,"input_uncached_usd":0.005030000000000001,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0078740420285612}
{"item_id":"semeval2013.d006.s013.t007","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":2,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":3,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":4,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":9,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":912,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.007260000000000001,"input_uncached_usd":0.004560000000000001,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.296331709017977}
{"item_id":"semeval2013.d006.s013.t009","gold_sense_keys":["referee%1:18:00::"],"candidates":[{"index":1,"sense_key":"referee%1:18:01::","synset_id":"reviewer.n.01"},{"index":2,"sense_key":"referee%1:18:02::","synset_id":"referee.n.03"},{"index":3,"sense_key":"referee%1:18:00::","synset_id":"referee.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"referee%1:18:00::","call_ids":["semeval2013.d006.s013.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"referee%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":36,"reasoning_output_tokens":21},"cost":{"currency":"USD","total_usd":0.003895,"input_uncached_usd":0.0028150000000000002,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.1837713329587132}
{"item_id":"semeval2013.d006.s014.t001","gold_sense_keys":["complaint%1:10:00::"],"candidates":[{"index":1,"sense_key":"complaint%1:10:04::","synset_id":"charge.n.02"},{"index":2,"sense_key":"complaint%1:10:00::","synset_id":"complaint.n.03"},{"index":3,"sense_key":"complaint%1:10:02::","synset_id":"complaint.n.02"},{"index":4,"sense_key":"complaint%1:10:03::","synset_id":"complaint.n.04"},{"index":5,"sense_key":"complaint%1:26:00::","synset_id":"ailment.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"complaint%1:10:00::","call_ids":["semeval2013.d006.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"complaint%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.00482,"input_uncached_usd":0.00287,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7078500000061467}
{"item_id":"semeval2013.d006.s015.t000","gold_sense_keys":["question%1:10:00::","question%1:10:01::","question%1:10:03::"],"candidates":[{"index":1,"sense_key":"question%1:10:04::","synset_id":"question.n.06"},{"index":2,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":3,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"},{"index":4,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"},{"index":5,"sense_key":"question%1:10:00::","synset_id":"question.n.01"},{"index":6,"sense_key":"question%1:10:03::","synset_id":"question.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"question%1:10:01::","call_ids":["semeval2013.d006.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"question%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":671,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":113},"cost":{"currency":"USD","total_usd":0.0072250000000000005,"input_uncached_usd":0.0033550000000000003,"input_cached_usd":0.0,"output_usd":0.00387,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.71093075000681}
{"item_id":"semeval2013.d006.s015.t001","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":2,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":3,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":4,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":5,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":6,"sense_key":"type%1:06:00::","synset_id":"type.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2013.d006.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.0046,"input_uncached_usd":0.0030700000000000002,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5293529590126127}
{"item_id":"semeval2013.d006.s015.t002","gold_sense_keys":["advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"},{"index":2,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"},{"index":3,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"advantage%1:07:01::","call_ids":["semeval2013.d006.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"advantage%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":196,"reasoning_output_tokens":180},"cost":{"currency":"USD","total_usd":0.008350000000000002,"input_uncached_usd":0.0024700000000000004,"input_cached_usd":0.0,"output_usd":0.00588,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.598333458066918}
{"item_id":"semeval2013.d006.s015.t004","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":2,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":3,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":4,"sense_key":"action%1:04:00::","synset_id":"action.n.10"},{"index":5,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":6,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":7,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":8,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":9,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":10,"sense_key":"action%1:26:00::","synset_id":"action.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.0066,"input_uncached_usd":0.00417,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.001649499987252}
{"item_id":"semeval2013.d006.s016.t000","gold_sense_keys":["situation%1:26:00::","situation%1:26:01::"],"candidates":[{"index":1,"sense_key":"situation%1:04:00::","synset_id":"position.n.06"},{"index":2,"sense_key":"situation%1:15:00::","synset_id":"site.n.02"},{"index":3,"sense_key":"situation%1:26:02::","synset_id":"situation.n.03"},{"index":4,"sense_key":"situation%1:26:01::","synset_id":"situation.n.02"},{"index":5,"sense_key":"situation%1:26:00::","synset_id":"situation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"situation%1:26:01::","call_ids":["semeval2013.d006.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"situation%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":632,"cached_input_tokens":0,"output_tokens":242,"reasoning_output_tokens":226},"cost":{"currency":"USD","total_usd":0.01042,"input_uncached_usd":0.00316,"input_cached_usd":0.0,"output_usd":0.00726,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8056249999208376}
{"item_id":"semeval2013.d006.s016.t002","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":2,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"},{"index":3,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"benefit%1:07:00::","call_ids":["semeval2013.d006.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.004655,"input_uncached_usd":0.0023750000000000004,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8505201250081882}
{"item_id":"semeval2013.d006.s016.t003","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":2,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":3,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":4,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":5,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":6,"sense_key":"game%1:04:04::","synset_id":"game.n.11"},{"index":7,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":10,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":11,"sense_key":"game%1:05:00::","synset_id":"game.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d006.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":810,"cached_input_tokens":0,"output_tokens":165,"reasoning_output_tokens":149},"cost":{"currency":"USD","total_usd":0.009000000000000001,"input_uncached_usd":0.004050000000000001,"input_cached_usd":0.0,"output_usd":0.00495,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.419893749989569}
{"item_id":"semeval2013.d006.s016.t004","gold_sense_keys":["result%1:19:00::"],"candidates":[{"index":1,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":2,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"result%1:19:00::","call_ids":["semeval2013.d006.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"result%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":137},"cost":{"currency":"USD","total_usd":0.007575,"input_uncached_usd":0.0029850000000000002,"input_cached_usd":0.0,"output_usd":0.00459,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.499906125012785}
{"item_id":"semeval2013.d006.s016.t005","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":4,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":5,"sense_key":"action%1:04:00::","synset_id":"action.n.10"},{"index":6,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":7,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":8,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":9,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":10,"sense_key":"action%1:10:00::","synset_id":"action.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":844,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.006950000000000001,"input_uncached_usd":0.004220000000000001,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.344321958022192}
{"item_id":"semeval2013.d006.s016.t006","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:01::","synset_id":"team.n.02"},{"index":2,"sense_key":"team%1:14:00::","synset_id":"team.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.003955,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.053791166981682}
{"item_id":"semeval2013.d006.s016.t007","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:03::","synset_id":"player.n.05"},{"index":2,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":3,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:01::","synset_id":"player.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s016.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.0045850000000000005,"input_uncached_usd":0.002785,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.506118749966845}
{"item_id":"semeval2013.d006.s017.t000","gold_sense_keys":["strategy%1:09:00::"],"candidates":[{"index":1,"sense_key":"strategy%1:09:01::","synset_id":"strategy.n.02"},{"index":2,"sense_key":"strategy%1:09:00::","synset_id":"scheme.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"strategy%1:09:00::","call_ids":["semeval2013.d006.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"strategy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00392,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5907636670162901}
{"item_id":"semeval2013.d006.s017.t001","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":2,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":3,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":4,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":5,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":6,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":9,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":10,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d006.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":121},"cost":{"currency":"USD","total_usd":0.008125,"input_uncached_usd":0.004015,"input_cached_usd":0.0,"output_usd":0.00411,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.40930437494535}
{"item_id":"semeval2013.d006.s017.t003","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":2,"sense_key":"player%1:18:03::","synset_id":"player.n.05"},{"index":3,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":4,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":5,"sense_key":"player%1:18:01::","synset_id":"player.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":550,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.00485,"input_uncached_usd":0.0027500000000000003,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0466850000666454}
{"item_id":"semeval2013.d006.s018.t000","gold_sense_keys":["advantage%1:07:00::","advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"},{"index":2,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"},{"index":3,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"advantage%1:07:01::","call_ids":["semeval2013.d006.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"advantage%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":292,"reasoning_output_tokens":277},"cost":{"currency":"USD","total_usd":0.011370000000000002,"input_uncached_usd":0.0026100000000000003,"input_cached_usd":0.0,"output_usd":0.00876,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.7296829170081764}
{"item_id":"semeval2013.d006.s018.t001","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"},{"index":2,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":3,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":4,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fact%1:09:01::","call_ids":["semeval2013.d006.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fact%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":580,"cached_input_tokens":0,"output_tokens":469,"reasoning_output_tokens":453},"cost":{"currency":"USD","total_usd":0.01697,"input_uncached_usd":0.0029000000000000002,"input_cached_usd":0.0,"output_usd":0.014070000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.328517209040001}
{"item_id":"semeval2013.d006.s018.t002","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:03::","synset_id":"player.n.05"},{"index":4,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":5,"sense_key":"player%1:18:04::","synset_id":"player.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.004595,"input_uncached_usd":0.0028850000000000004,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5633008329896256}
{"item_id":"semeval2013.d006.s018.t003","gold_sense_keys":["phase%1:28:00::"],"candidates":[{"index":1,"sense_key":"phase%1:07:00::","synset_id":"phase.n.04"},{"index":2,"sense_key":"phase%1:26:00::","synset_id":"phase.n.02"},{"index":3,"sense_key":"phase%1:28:00::","synset_id":"phase.n.01"},{"index":4,"sense_key":"phase%1:28:01::","synset_id":"phase.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"phase%1:28:00::","call_ids":["semeval2013.d006.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"phase%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.004620000000000001,"input_uncached_usd":0.0030900000000000003,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.365096916910261}
{"item_id":"semeval2013.d006.s018.t004","gold_sense_keys":["championship%1:11:00::"],"candidates":[{"index":1,"sense_key":"championship%1:26:00::","synset_id":"championship.n.01"},{"index":2,"sense_key":"championship%1:11:00::","synset_id":"championship.n.02"},{"index":3,"sense_key":"championship%1:04:00::","synset_id":"backing.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"championship%1:11:00::","call_ids":["semeval2013.d006.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"championship%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.004825,"input_uncached_usd":0.002605,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2951766669284552}
{"item_id":"semeval2013.d006.s018.t006","gold_sense_keys":["participation%1:04:00::"],"candidates":[{"index":1,"sense_key":"participation%1:26:00::","synset_id":"participation.n.02"},{"index":2,"sense_key":"participation%1:04:00::","synset_id":"engagement.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"participation%1:04:00::","call_ids":["semeval2013.d006.s018.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"participation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.004920000000000001,"input_uncached_usd":0.00249,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.869154499960132}
{"item_id":"semeval2013.d006.s018.t007","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:04:04::","synset_id":"game.n.11"},{"index":6,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":7,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":10,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":11,"sense_key":"game%1:06:00::","synset_id":"game.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d006.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":829,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.007295,"input_uncached_usd":0.004145,"input_cached_usd":0.0,"output_usd":0.00315,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.816253041033633}
{"item_id":"semeval2013.d006.s019.t000","gold_sense_keys":["term%1:28:00::"],"candidates":[{"index":1,"sense_key":"term%1:09:00::","synset_id":"term.n.04"},{"index":2,"sense_key":"term%1:10:02::","synset_id":"condition.n.07"},{"index":3,"sense_key":"term%1:10:00::","synset_id":"term.n.01"},{"index":4,"sense_key":"term%1:10:01::","synset_id":"term.n.05"},{"index":5,"sense_key":"term%1:06:00::","synset_id":"terminus.n.03"},{"index":6,"sense_key":"term%1:28:00::","synset_id":"term.n.02"},{"index":7,"sense_key":"term%1:28:01::","synset_id":"term.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"term%1:28:00::","call_ids":["semeval2013.d006.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"term%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.0055000000000000005,"input_uncached_usd":0.0033100000000000004,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.563130250084214}
{"item_id":"semeval2013.d006.s019.t001","gold_sense_keys":["advantage%1:07:00::","advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"},{"index":2,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"},{"index":3,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"advantage%1:07:01::","call_ids":["semeval2013.d006.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"advantage%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":240,"reasoning_output_tokens":224},"cost":{"currency":"USD","total_usd":0.009585000000000002,"input_uncached_usd":0.0023850000000000004,"input_cached_usd":0.0,"output_usd":0.0072,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.369515958009288}
{"item_id":"semeval2013.d006.s019.t002","gold_sense_keys":["degree%1:07:00::","degree%1:26:01::"],"candidates":[{"index":1,"sense_key":"degree%1:23:03::","synset_id":"degree.n.06"},{"index":2,"sense_key":"degree%1:09:00::","synset_id":"degree.n.05"},{"index":3,"sense_key":"degree%1:10:00::","synset_id":"academic_degree.n.01"},{"index":4,"sense_key":"degree%1:26:01::","synset_id":"degree.n.02"},{"index":5,"sense_key":"degree%1:07:01::","synset_id":"degree.n.07"},{"index":6,"sense_key":"degree%1:07:00::","synset_id":"degree.n.01"},{"index":7,"sense_key":"degree%1:23:00::","synset_id":"degree.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"degree%1:07:00::","call_ids":["semeval2013.d006.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"degree%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":2428,"reasoning_output_tokens":2388},"cost":{"currency":"USD","total_usd":0.076165,"input_uncached_usd":0.0033250000000000003,"input_cached_usd":0.0,"output_usd":0.07284,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":37.98376562492922}
{"item_id":"semeval2013.d006.s019.t004","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:01::","synset_id":"team.n.02"},{"index":2,"sense_key":"team%1:14:00::","synset_id":"team.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":34,"reasoning_output_tokens":19},"cost":{"currency":"USD","total_usd":0.003105,"input_uncached_usd":0.002085,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3098252500640228}
{"item_id":"semeval2013.d006.s019.t005","gold_sense_keys":["change%1:11:00::"],"candidates":[{"index":1,"sense_key":"change%1:24:00::","synset_id":"change.n.02"},{"index":2,"sense_key":"change%1:11:00::","synset_id":"change.n.01"},{"index":3,"sense_key":"change%1:04:00::","synset_id":"change.n.03"},{"index":4,"sense_key":"change%1:06:01::","synset_id":"change.n.06"},{"index":5,"sense_key":"change%1:21:01::","synset_id":"change.n.08"},{"index":6,"sense_key":"change%1:07:00::","synset_id":"variety.n.06"},{"index":7,"sense_key":"change%1:21:02::","synset_id":"change.n.05"},{"index":8,"sense_key":"change%1:06:00::","synset_id":"change.n.07"},{"index":9,"sense_key":"change%1:21:03::","synset_id":"change.n.09"},{"index":10,"sense_key":"change%1:19:00::","synset_id":"change.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"change%1:11:00::","call_ids":["semeval2013.d006.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"change%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":787,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1025},"cost":{"currency":"USD","total_usd":0.035315,"input_uncached_usd":0.003935,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":19.296708708978258}
{"item_id":"semeval2013.d006.s019.t007","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:00::","synset_id":"action.n.10"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":6,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":7,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":8,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":9,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":10,"sense_key":"action%1:07:00::","synset_id":"action.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":818,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.0068200000000000005,"input_uncached_usd":0.004090000000000001,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.908899208996445}
{"item_id":"semeval2013.d006.s019.t008","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:03::","synset_id":"player.n.05"},{"index":2,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":3,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s019.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.00455,"input_uncached_usd":0.00266,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8300069579854608}
{"item_id":"semeval2013.d006.s019.t010","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:04::","synset_id":"game.n.11"},{"index":2,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":3,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":4,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":5,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":6,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":7,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":8,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":9,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:23:00::","synset_id":"game.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d006.s019.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":785,"cached_input_tokens":0,"output_tokens":174,"reasoning_output_tokens":158},"cost":{"currency":"USD","total_usd":0.009145,"input_uncached_usd":0.0039250000000000005,"input_cached_usd":0.0,"output_usd":0.00522,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.583380958996713}
{"item_id":"semeval2013.d006.s019.t013","gold_sense_keys":["suspension%1:04:01::"],"candidates":[{"index":1,"sense_key":"suspension%1:04:00::","synset_id":"suspension.n.06"},{"index":2,"sense_key":"suspension%1:28:00::","synset_id":"pause.n.01"},{"index":3,"sense_key":"suspension%1:11:00::","synset_id":"suspension.n.04"},{"index":4,"sense_key":"suspension%1:26:00::","synset_id":"abeyance.n.01"},{"index":5,"sense_key":"suspension%1:06:00::","synset_id":"suspension.n.05"},{"index":6,"sense_key":"suspension%1:27:00::","synset_id":"suspension.n.01"},{"index":7,"sense_key":"suspension%1:04:01::","synset_id":"suspension.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"suspension%1:04:01::","call_ids":["semeval2013.d006.s019.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"suspension%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":641,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.005365,"input_uncached_usd":0.0032050000000000004,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1428820829605684}
{"item_id":"semeval2013.d006.s019.t015","gold_sense_keys":["accumulation%1:04:00::"],"candidates":[{"index":1,"sense_key":"accumulation%1:22:00::","synset_id":"accretion.n.01"},{"index":2,"sense_key":"accumulation%1:14:00::","synset_id":"collection.n.01"},{"index":3,"sense_key":"accumulation%1:04:00::","synset_id":"accumulation.n.03"},{"index":4,"sense_key":"accumulation%1:21:00::","synset_id":"accumulation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"accumulation%1:04:00::","call_ids":["semeval2013.d006.s019.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"accumulation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":500,"reasoning_output_tokens":485},"cost":{"currency":"USD","total_usd":0.017515000000000003,"input_uncached_usd":0.0025150000000000003,"input_cached_usd":0.0,"output_usd":0.015000000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.328593042097054}
{"item_id":"semeval2013.d006.s020.t000","gold_sense_keys":["round%1:28:01::"],"candidates":[{"index":1,"sense_key":"round%1:13:00::","synset_id":"round.n.10"},{"index":2,"sense_key":"round%1:15:00::","synset_id":"beat.n.01"},{"index":3,"sense_key":"round%1:28:01::","synset_id":"cycle.n.01"},{"index":4,"sense_key":"round%1:10:01::","synset_id":"round.n.12"},{"index":5,"sense_key":"round%1:04:03::","synset_id":"round_of_golf.n.01"},{"index":6,"sense_key":"round%1:04:01::","synset_id":"round.n.06"},{"index":7,"sense_key":"round%1:06:01::","synset_id":"round.n.01"},{"index":8,"sense_key":"round%1:10:00::","synset_id":"round.n.11"},{"index":9,"sense_key":"round%1:06:03::","synset_id":"rung.n.01"},{"index":10,"sense_key":"round%1:13:01::","synset_id":"round.n.09"},{"index":11,"sense_key":"round%1:17:00::","synset_id":"round.n.08"},{"index":12,"sense_key":"round%1:28:00::","synset_id":"turn.n.09"},{"index":13,"sense_key":"round%1:04:00::","synset_id":"round.n.04"},{"index":14,"sense_key":"round%1:06:00::","synset_id":"circle.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"round%1:28:00::","call_ids":["semeval2013.d006.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"round%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":921,"cached_input_tokens":0,"output_tokens":1215,"reasoning_output_tokens":1187},"cost":{"currency":"USD","total_usd":0.041055,"input_uncached_usd":0.004605000000000001,"input_cached_usd":0.0,"output_usd":0.03645,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":20.10943616600707}
{"item_id":"semeval2013.d006.s020.t001","gold_sense_keys":["advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"},{"index":2,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"},{"index":3,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"advantage%1:07:01::","call_ids":["semeval2013.d006.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"advantage%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":257,"reasoning_output_tokens":241},"cost":{"currency":"USD","total_usd":0.010165,"input_uncached_usd":0.002455,"input_cached_usd":0.0,"output_usd":0.00771,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.127490375074558}
{"item_id":"semeval2013.d006.s021.t000","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":2,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":3,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":4,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":5,"sense_key":"action%1:04:00::","synset_id":"action.n.10"},{"index":6,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":7,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":8,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":9,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":10,"sense_key":"action%1:06:00::","synset_id":"action.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.007380000000000001,"input_uncached_usd":0.004260000000000001,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6799918750766665}
{"item_id":"semeval2013.d006.s021.t002","gold_sense_keys":["part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":2,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":3,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":4,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":5,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":6,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":7,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":8,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":9,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":10,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":11,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"},{"index":12,"sense_key":"part%1:09:02::","synset_id":"part.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"part%1:24:00::","call_ids":["semeval2013.d006.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"part%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":988,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02078,"input_uncached_usd":0.004940000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.326608625007793}
{"item_id":"semeval2013.d006.s021.t003","gold_sense_keys":["decision%1:04:00::"],"candidates":[{"index":1,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"},{"index":2,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"},{"index":3,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"},{"index":4,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"},{"index":5,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decision%1:09:00::","call_ids":["semeval2013.d006.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decision%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018915,"input_uncached_usd":0.003075,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.882307542022318}
{"item_id":"semeval2013.d006.s021.t005","gold_sense_keys":["rule%1:10:02::"],"candidates":[{"index":1,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":2,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":3,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":4,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":5,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":6,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":7,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":8,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":9,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":10,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"},{"index":11,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":12,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"rule%1:10:02::","call_ids":["semeval2013.d006.s021.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"rule%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":945,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":121},"cost":{"currency":"USD","total_usd":0.008834999999999999,"input_uncached_usd":0.004725,"input_cached_usd":0.0,"output_usd":0.00411,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.561924208072014}
{"item_id":"semeval2013.d006.s022.t000","gold_sense_keys":["assumption%1:09:00::"],"candidates":[{"index":1,"sense_key":"assumption%1:04:00::","synset_id":"assumption.n.03"},{"index":2,"sense_key":"assumption%1:11:00::","synset_id":"assumption.n.05"},{"index":3,"sense_key":"assumption%1:07:00::","synset_id":"presumption.n.03"},{"index":4,"sense_key":"assumption%1:09:00::","synset_id":"assumption.n.02"},{"index":5,"sense_key":"assumption%1:10:00::","synset_id":"premise.n.01"},{"index":6,"sense_key":"assumption%1:04:01::","synset_id":"assumption.n.07"},{"index":7,"sense_key":"assumption%1:28:00::","synset_id":"assumption.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"assumption%1:10:00::","call_ids":["semeval2013.d006.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"assumption%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":720,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01944,"input_uncached_usd":0.0036000000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.943760458030738}
{"item_id":"semeval2013.d006.s022.t001","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":2,"sense_key":"player%1:18:03::","synset_id":"player.n.05"},{"index":3,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":4,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":5,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.0045000000000000005,"input_uncached_usd":0.00267,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.711481541977264}
{"item_id":"semeval2013.d006.s022.t002","gold_sense_keys":["application%1:04:00::"],"candidates":[{"index":1,"sense_key":"application%1:04:01::","synset_id":"application.n.06"},{"index":2,"sense_key":"application%1:04:00::","synset_id":"application.n.07"},{"index":3,"sense_key":"application%1:04:02::","synset_id":"application.n.01"},{"index":4,"sense_key":"application%1:06:00::","synset_id":"lotion.n.02"},{"index":5,"sense_key":"application%1:10:01::","synset_id":"application.n.04"},{"index":6,"sense_key":"application%1:04:03::","synset_id":"application.n.03"},{"index":7,"sense_key":"application%1:10:00::","synset_id":"application.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"application%1:04:00::","call_ids":["semeval2013.d006.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"application%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":679,"cached_input_tokens":0,"output_tokens":230,"reasoning_output_tokens":214},"cost":{"currency":"USD","total_usd":0.010295000000000002,"input_uncached_usd":0.0033950000000000004,"input_cached_usd":0.0,"output_usd":0.0069,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8036132500274107}
{"item_id":"semeval2013.d006.s022.t004","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":2,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":3,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":4,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":7,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":8,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":9,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":10,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":11,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":12,"sense_key":"case%1:06:01::","synset_id":"case.n.20"},{"index":13,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":14,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":15,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":16,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":17,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":18,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":19,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":20,"sense_key":"case%1:10:03::","synset_id":"font.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"case%1:11:00::","call_ids":["semeval2013.d006.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"case%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1225,"cached_input_tokens":0,"output_tokens":159,"reasoning_output_tokens":143},"cost":{"currency":"USD","total_usd":0.010895000000000002,"input_uncached_usd":0.006125,"input_cached_usd":0.0,"output_usd":0.00477,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6571760420920327}
{"item_id":"semeval2013.d006.s023.t000","gold_sense_keys":["decision%1:04:00::"],"candidates":[{"index":1,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"},{"index":2,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"},{"index":3,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"},{"index":4,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"},{"index":5,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"decision%1:04:00::","call_ids":["semeval2013.d006.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"decision%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":348,"reasoning_output_tokens":332},"cost":{"currency":"USD","total_usd":0.013375,"input_uncached_usd":0.002935,"input_cached_usd":0.0,"output_usd":0.01044,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.694707834045403}
{"item_id":"semeval2013.d006.s024.t002","gold_sense_keys":["law%1:14:00::"],"candidates":[{"index":1,"sense_key":"law%1:14:01::","synset_id":"police.n.01"},{"index":2,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":3,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":4,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":5,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":6,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":7,"sense_key":"law%1:09:01::","synset_id":"law.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"law%1:14:00::","call_ids":["semeval2013.d006.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"law%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":686,"cached_input_tokens":0,"output_tokens":273,"reasoning_output_tokens":257},"cost":{"currency":"USD","total_usd":0.01162,"input_uncached_usd":0.0034300000000000003,"input_cached_usd":0.0,"output_usd":0.00819,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.999774332973175}
{"item_id":"semeval2013.d006.s024.t003","gold_sense_keys":["fraud%1:04:01::"],"candidates":[{"index":1,"sense_key":"fraud%1:04:00::","synset_id":"fraud.n.01"},{"index":2,"sense_key":"fraud%1:04:01::","synset_id":"fraud.n.03"},{"index":3,"sense_key":"fraud%1:18:00::","synset_id":"imposter.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fraud%1:04:01::","call_ids":["semeval2013.d006.s024.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fraud%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":1564,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.04936,"input_uncached_usd":0.0024400000000000003,"input_cached_usd":0.0,"output_usd":0.04692,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":28.002307625021785}
{"item_id":"semeval2013.d006.s024.t004","gold_sense_keys":["occasion%1:11:00::"],"candidates":[{"index":1,"sense_key":"occasion%1:28:00::","synset_id":"occasion.n.04"},{"index":2,"sense_key":"occasion%1:26:00::","synset_id":"occasion.n.05"},{"index":3,"sense_key":"occasion%1:11:01::","synset_id":"affair.n.03"},{"index":4,"sense_key":"occasion%1:11:00::","synset_id":"juncture.n.01"},{"index":5,"sense_key":"occasion%1:16:00::","synset_id":"occasion.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"occasion%1:11:00::","call_ids":["semeval2013.d006.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"occasion%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01869,"input_uncached_usd":0.00285,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.496397959068418}
{"item_id":"semeval2013.d006.s024.t005","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":2,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":3,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":4,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":5,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":6,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":7,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":8,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":9,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":10,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":11,"sense_key":"case%1:06:01::","synset_id":"case.n.20"},{"index":12,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":13,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":14,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":15,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":16,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":19,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":20,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"case%1:11:00::","call_ids":["semeval2013.d006.s024.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"case%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1230,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.00936,"input_uncached_usd":0.00615,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5391874159686267}
{"item_id":"semeval2013.d006.s025.t001","gold_sense_keys":["factor%1:11:00::"],"candidates":[{"index":1,"sense_key":"factor%1:09:01::","synset_id":"factor.n.06"},{"index":2,"sense_key":"factor%1:11:00::","synset_id":"factor.n.01"},{"index":3,"sense_key":"factor%1:23:00::","synset_id":"divisor.n.01"},{"index":4,"sense_key":"factor%1:09:00::","synset_id":"component.n.01"},{"index":5,"sense_key":"factor%1:23:01::","synset_id":"factor.n.05"},{"index":6,"sense_key":"factor%1:08:00::","synset_id":"gene.n.01"},{"index":7,"sense_key":"factor%1:18:00::","synset_id":"agent.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"factor%1:11:00::","call_ids":["semeval2013.d006.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"factor%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":627,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":80},"cost":{"currency":"USD","total_usd":0.006015,"input_uncached_usd":0.003135,"input_cached_usd":0.0,"output_usd":0.00288,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.17899008304812}
{"item_id":"semeval2013.d006.s025.t002","gold_sense_keys":["expulsion%1:04:01::"],"candidates":[{"index":1,"sense_key":"expulsion%1:04:00::","synset_id":"expulsion.n.03"},{"index":2,"sense_key":"expulsion%1:04:02::","synset_id":"extrusion.n.02"},{"index":3,"sense_key":"expulsion%1:04:01::","synset_id":"ejection.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"expulsion%1:04:01::","call_ids":["semeval2013.d006.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"expulsion%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":186,"reasoning_output_tokens":170},"cost":{"currency":"USD","total_usd":0.007930000000000001,"input_uncached_usd":0.00235,"input_cached_usd":0.0,"output_usd":0.00558,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9010148749221116}
{"item_id":"semeval2013.d006.s025.t003","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":2,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":3,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":4,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":7,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":8,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":9,"sense_key":"way%1:21:00::","synset_id":"way.n.12"},{"index":10,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":11,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":12,"sense_key":"way%1:26:00::","synset_id":"way.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"way%1:07:01::","call_ids":["semeval2013.d006.s025.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":793,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":113},"cost":{"currency":"USD","total_usd":0.007835000000000002,"input_uncached_usd":0.003965000000000001,"input_cached_usd":0.0,"output_usd":0.00387,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3597948329988867}
{"item_id":"semeval2013.d006.s025.t004","gold_sense_keys":["transmission%1:04:01::"],"candidates":[{"index":1,"sense_key":"transmission%1:11:00::","synset_id":"infection.n.04"},{"index":2,"sense_key":"transmission%1:06:01::","synset_id":"transmission.n.05"},{"index":3,"sense_key":"transmission%1:04:01::","synset_id":"transmission.n.01"},{"index":4,"sense_key":"transmission%1:10:00::","synset_id":"transmission.n.02"},{"index":5,"sense_key":"transmission%1:23:00::","synset_id":"transmittance.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"transmission%1:04:01::","call_ids":["semeval2013.d006.s025.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"transmission%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.004995,"input_uncached_usd":0.002475,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8944112920435145}
{"item_id":"semeval2013.d006.s026.t000","gold_sense_keys":["damage%1:11:00::"],"candidates":[{"index":1,"sense_key":"damage%1:11:00::","synset_id":"damage.n.01"},{"index":2,"sense_key":"damage%1:11:01::","synset_id":"damage.n.02"},{"index":3,"sense_key":"damage%1:21:00::","synset_id":"price.n.02"},{"index":4,"sense_key":"damage%1:04:00::","synset_id":"damage.n.03"},{"index":5,"sense_key":"damage%1:04:01::","synset_id":"wrong.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%1:11:00::","call_ids":["semeval2013.d006.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":96},"cost":{"currency":"USD","total_usd":0.0057150000000000005,"input_uncached_usd":0.0023850000000000004,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3399685829645023}
{"item_id":"semeval2013.d006.s026.t001","gold_sense_keys":["principle%1:09:00::"],"candidates":[{"index":1,"sense_key":"principle%1:09:04::","synset_id":"principle.n.04"},{"index":2,"sense_key":"principle%1:09:03::","synset_id":"principle.n.01"},{"index":3,"sense_key":"principle%1:09:02::","synset_id":"rationale.n.01"},{"index":4,"sense_key":"principle%1:10:00::","synset_id":"principle.n.05"},{"index":5,"sense_key":"principle%1:09:01::","synset_id":"principle.n.02"},{"index":6,"sense_key":"principle%1:09:00::","synset_id":"principle.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"principle%1:09:01::","call_ids":["semeval2013.d006.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"principle%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":571,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018695,"input_uncached_usd":0.0028550000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.81469799997285}
{"item_id":"semeval2013.d006.s026.t003","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:00::","synset_id":"action.n.10"},{"index":4,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":5,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":8,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":9,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":10,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s026.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.005935,"input_uncached_usd":0.0038050000000000002,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6509655829286203}
{"item_id":"semeval2013.d007.s000.t001","gold_sense_keys":["possibility%1:26:00::"],"candidates":[{"index":1,"sense_key":"possibility%1:26:00::","synset_id":"possibility.n.02"},{"index":2,"sense_key":"possibility%1:09:00::","synset_id":"hypothesis.n.02"},{"index":3,"sense_key":"possibility%1:09:01::","synset_id":"possibility.n.04"},{"index":4,"sense_key":"possibility%1:09:03::","synset_id":"possibility.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"possibility%1:26:00::","call_ids":["semeval2013.d007.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"possibility%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":392,"cached_input_tokens":0,"output_tokens":244,"reasoning_output_tokens":228},"cost":{"currency":"USD","total_usd":0.009280000000000002,"input_uncached_usd":0.0019600000000000004,"input_cached_usd":0.0,"output_usd":0.00732,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.320528666023165}
{"item_id":"semeval2013.d007.s000.t002","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":2,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":3,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":4,"sense_key":"type%1:06:00::","synset_id":"type.n.06"},{"index":5,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":6,"sense_key":"type%1:09:00::","synset_id":"type.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2013.d007.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.004015,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5847270840313286}
{"item_id":"semeval2013.d007.s001.t000","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":2,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":3,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"},{"index":4,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"discovery%1:09:00::","call_ids":["semeval2013.d007.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"discovery%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":358,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01763,"input_uncached_usd":0.0017900000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.071098999935202}
{"item_id":"semeval2013.d007.s001.t002","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":364,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.0041,"input_uncached_usd":0.0018200000000000002,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8404507500818}
{"item_id":"semeval2013.d007.s001.t003","gold_sense_keys":["nutrient%1:03:00::"],"candidates":[{"index":1,"sense_key":"nutrient%1:03:00::","synset_id":"food.n.01"},{"index":2,"sense_key":"nutrient%1:03:01::","synset_id":"nutrient.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"nutrient%1:03:01::","call_ids":["semeval2013.d007.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"nutrient%1:03:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":314,"cached_input_tokens":0,"output_tokens":256,"reasoning_output_tokens":240},"cost":{"currency":"USD","total_usd":0.009250000000000001,"input_uncached_usd":0.0015700000000000002,"input_cached_usd":0.0,"output_usd":0.00768,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.52802912495099}
{"item_id":"semeval2013.d007.s001.t004","gold_sense_keys":["scope%1:07:00::"],"candidates":[{"index":1,"sense_key":"scope%1:06:02::","synset_id":"telescope.n.01"},{"index":2,"sense_key":"scope%1:26:00::","synset_id":"setting.n.02"},{"index":3,"sense_key":"scope%1:07:00::","synset_id":"scope.n.01"},{"index":4,"sense_key":"scope%1:06:01::","synset_id":"oscilloscope.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"scope%1:07:00::","call_ids":["semeval2013.d007.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"scope%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":408,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00384,"input_uncached_usd":0.00204,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.063393959077075}
{"item_id":"semeval2013.d007.s001.t005","gold_sense_keys":["form%1:09:01::"],"candidates":[{"index":1,"sense_key":"form%1:03:00::","synset_id":"shape.n.02"},{"index":2,"sense_key":"form%1:09:00::","synset_id":"form.n.03"},{"index":3,"sense_key":"form%1:10:02::","synset_id":"form.n.10"},{"index":4,"sense_key":"form%1:07:03::","synset_id":"form.n.11"},{"index":5,"sense_key":"form%1:06:01::","synset_id":"form.n.16"},{"index":6,"sense_key":"form%1:07:01::","synset_id":"shape.n.01"},{"index":7,"sense_key":"form%1:09:01::","synset_id":"kind.n.01"},{"index":8,"sense_key":"form%1:10:00::","synset_id":"form.n.01"},{"index":9,"sense_key":"form%1:10:01::","synset_id":"form.n.08"},{"index":10,"sense_key":"form%1:26:00::","synset_id":"phase.n.02"},{"index":11,"sense_key":"form%1:07:00::","synset_id":"form.n.07"},{"index":12,"sense_key":"form%1:07:02::","synset_id":"form.n.14"},{"index":13,"sense_key":"form%1:14:02::","synset_id":"form.n.09"},{"index":14,"sense_key":"form%1:08:00::","synset_id":"human_body.n.01"},{"index":15,"sense_key":"form%1:14:00::","synset_id":"class.n.02"},{"index":16,"sense_key":"form%1:06:00::","synset_id":"mannequin.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"form%1:09:01::","call_ids":["semeval2013.d007.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"form%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":961,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":104},"cost":{"currency":"USD","total_usd":0.008405,"input_uncached_usd":0.004805,"input_cached_usd":0.0,"output_usd":0.0036,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4123878749087453}
{"item_id":"semeval2013.d007.s002.t002","gold_sense_keys":["earth%1:17:00::"],"candidates":[{"index":1,"sense_key":"earth%1:27:00::","synset_id":"earth.n.02"},{"index":2,"sense_key":"earth%1:17:01::","synset_id":"land.n.04"},{"index":3,"sense_key":"earth%1:09:00::","synset_id":"worldly_concern.n.01"},{"index":4,"sense_key":"earth%1:17:00::","synset_id":"earth.n.01"},{"index":5,"sense_key":"earth%1:15:00::","synset_id":"earth.n.04"},{"index":6,"sense_key":"earth%1:06:00::","synset_id":"ground.n.09"},{"index":7,"sense_key":"earth%1:27:01::","synset_id":"earth.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"earth%1:17:00::","call_ids":["semeval2013.d007.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"earth%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.005345000000000001,"input_uncached_usd":0.0028250000000000003,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8341691669775173}
{"item_id":"semeval2013.d007.s002.t003","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":2,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":3,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":7,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"},{"index":8,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":9,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":10,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":11,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":12,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":13,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":14,"sense_key":"life%1:26:01::","synset_id":"life.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019855,"input_uncached_usd":0.004015,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.683338000089861}
{"item_id":"semeval2013.d007.s002.t004","gold_sense_keys":["possibility%1:09:03::"],"candidates":[{"index":1,"sense_key":"possibility%1:26:00::","synset_id":"possibility.n.02"},{"index":2,"sense_key":"possibility%1:09:00::","synset_id":"hypothesis.n.02"},{"index":3,"sense_key":"possibility%1:09:03::","synset_id":"possibility.n.01"},{"index":4,"sense_key":"possibility%1:09:01::","synset_id":"possibility.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"possibility%1:09:03::","call_ids":["semeval2013.d007.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"possibility%1:09:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":273,"reasoning_output_tokens":257},"cost":{"currency":"USD","total_usd":0.010324999999999999,"input_uncached_usd":0.002135,"input_cached_usd":0.0,"output_usd":0.00819,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.158867249963805}
{"item_id":"semeval2013.d007.s002.t005","gold_sense_keys":["element%1:27:00::"],"candidates":[{"index":1,"sense_key":"element%1:15:00::","synset_id":"element.n.04"},{"index":2,"sense_key":"element%1:27:00::","synset_id":"chemical_element.n.01"},{"index":3,"sense_key":"element%1:27:01::","synset_id":"element.n.05"},{"index":4,"sense_key":"element%1:26:00::","synset_id":"element.n.06"},{"index":5,"sense_key":"element%1:25:00::","synset_id":"element.n.07"},{"index":6,"sense_key":"element%1:06:00::","synset_id":"component.n.03"},{"index":7,"sense_key":"element%1:09:00::","synset_id":"component.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"element%1:27:00::","call_ids":["semeval2013.d007.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"element%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":554,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.00478,"input_uncached_usd":0.0027700000000000003,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.594511750037782}
{"item_id":"semeval2013.d007.s003.t000","gold_sense_keys":["organism%1:03:00::"],"candidates":[{"index":1,"sense_key":"organism%1:14:00::","synset_id":"organism.n.02"},{"index":2,"sense_key":"organism%1:03:00::","synset_id":"organism.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"organism%1:03:00::","call_ids":["semeval2013.d007.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"organism%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":320,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.00292,"input_uncached_usd":0.0016,"input_cached_usd":0.0,"output_usd":0.00132,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.120531833032146}
{"item_id":"semeval2013.d007.s003.t001","gold_sense_keys":["capability%1:07:00::"],"candidates":[{"index":1,"sense_key":"capability%1:26:00::","synset_id":"capability.n.02"},{"index":2,"sense_key":"capability%1:09:00::","synset_id":"capability.n.03"},{"index":3,"sense_key":"capability%1:07:00::","synset_id":"capability.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"capability%1:07:00::","call_ids":["semeval2013.d007.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"capability%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":367,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017675,"input_uncached_usd":0.001835,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.044652792043053}
{"item_id":"semeval2013.d007.s004.t000","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"},{"index":2,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":406,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.0035,"input_uncached_usd":0.00203,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6217929580016062}
{"item_id":"semeval2013.d007.s004.t001","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.004085,"input_uncached_usd":0.0021650000000000003,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.388426125049591}
{"item_id":"semeval2013.d007.s005.t000","gold_sense_keys":["form%1:09:01::"],"candidates":[{"index":1,"sense_key":"form%1:10:00::","synset_id":"form.n.01"},{"index":2,"sense_key":"form%1:14:00::","synset_id":"class.n.02"},{"index":3,"sense_key":"form%1:26:00::","synset_id":"phase.n.02"},{"index":4,"sense_key":"form%1:14:02::","synset_id":"form.n.09"},{"index":5,"sense_key":"form%1:06:00::","synset_id":"mannequin.n.02"},{"index":6,"sense_key":"form%1:10:02::","synset_id":"form.n.10"},{"index":7,"sense_key":"form%1:07:03::","synset_id":"form.n.11"},{"index":8,"sense_key":"form%1:07:00::","synset_id":"form.n.07"},{"index":9,"sense_key":"form%1:07:01::","synset_id":"shape.n.01"},{"index":10,"sense_key":"form%1:08:00::","synset_id":"human_body.n.01"},{"index":11,"sense_key":"form%1:03:00::","synset_id":"shape.n.02"},{"index":12,"sense_key":"form%1:09:01::","synset_id":"kind.n.01"},{"index":13,"sense_key":"form%1:07:02::","synset_id":"form.n.14"},{"index":14,"sense_key":"form%1:09:00::","synset_id":"form.n.03"},{"index":15,"sense_key":"form%1:06:01::","synset_id":"form.n.16"},{"index":16,"sense_key":"form%1:10:01::","synset_id":"form.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"form%1:09:01::","call_ids":["semeval2013.d007.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"form%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1053,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.007635000000000001,"input_uncached_usd":0.005265000000000001,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2875399170443416}
{"item_id":"semeval2013.d007.s005.t001","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":2,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":3,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":4,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":5,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":6,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":7,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":8,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":11,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":12,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":13,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":888,"cached_input_tokens":0,"output_tokens":2075,"reasoning_output_tokens":2040},"cost":{"currency":"USD","total_usd":0.06669000000000001,"input_uncached_usd":0.00444,"input_cached_usd":0.0,"output_usd":0.06225,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":33.44391162507236}
{"item_id":"semeval2013.d007.s005.t002","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":2,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":3,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":6,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"},{"index":7,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2013.d007.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":90},"cost":{"currency":"USD","total_usd":0.006405000000000001,"input_uncached_usd":0.0032250000000000004,"input_cached_usd":0.0,"output_usd":0.00318,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.274315375019796}
{"item_id":"semeval2013.d007.s005.t004","gold_sense_keys":["edition%1:10:02::"],"candidates":[{"index":1,"sense_key":"edition%1:10:00::","synset_id":"edition.n.03"},{"index":2,"sense_key":"edition%1:09:00::","synset_id":"version.n.02"},{"index":3,"sense_key":"edition%1:14:00::","synset_id":"edition.n.02"},{"index":4,"sense_key":"edition%1:10:02::","synset_id":"edition.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"edition%1:10:02::","call_ids":["semeval2013.d007.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"edition%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":1825,"reasoning_output_tokens":1791},"cost":{"currency":"USD","total_usd":0.0572,"input_uncached_usd":0.0024500000000000004,"input_cached_usd":0.0,"output_usd":0.05475,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":25.24028766597621}
{"item_id":"semeval2013.d007.s005.t005","gold_sense_keys":["journal%1:10:00::"],"candidates":[{"index":1,"sense_key":"journal%1:06:01::","synset_id":"journal.n.04"},{"index":2,"sense_key":"journal%1:21:00::","synset_id":"daybook.n.01"},{"index":3,"sense_key":"journal%1:10:01::","synset_id":"diary.n.01"},{"index":4,"sense_key":"journal%1:10:00::","synset_id":"journal.n.02"},{"index":5,"sense_key":"journal%1:06:00::","synset_id":"journal.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"journal%1:10:00::","call_ids":["semeval2013.d007.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"journal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":466,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.00377,"input_uncached_usd":0.00233,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.412506083957851}
{"item_id":"semeval2013.d007.s006.t000","gold_sense_keys":["element%1:27:00::"],"candidates":[{"index":1,"sense_key":"element%1:27:01::","synset_id":"element.n.05"},{"index":2,"sense_key":"element%1:15:00::","synset_id":"element.n.04"},{"index":3,"sense_key":"element%1:09:00::","synset_id":"component.n.01"},{"index":4,"sense_key":"element%1:25:00::","synset_id":"element.n.07"},{"index":5,"sense_key":"element%1:26:00::","synset_id":"element.n.06"},{"index":6,"sense_key":"element%1:27:00::","synset_id":"chemical_element.n.01"},{"index":7,"sense_key":"element%1:06:00::","synset_id":"component.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"element%1:27:00::","call_ids":["semeval2013.d007.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"element%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.00528,"input_uncached_usd":0.0032400000000000003,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.425290249986574}
{"item_id":"semeval2013.d007.s006.t001","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":2,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":3,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":4,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":5,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":6,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"},{"index":7,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":8,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":9,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":10,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":11,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":12,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":13,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":14,"sense_key":"life%1:09:00::","synset_id":"life.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"life%1:26:00::","call_ids":["semeval2013.d007.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"life%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":897,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020325,"input_uncached_usd":0.004485,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.38365979201626}
{"item_id":"semeval2013.d007.s006.t002","gold_sense_keys":["carbon%1:27:00::"],"candidates":[{"index":1,"sense_key":"carbon%1:27:00::","synset_id":"carbon.n.01"},{"index":2,"sense_key":"carbon%1:06:00::","synset_id":"carbon.n.03"},{"index":3,"sense_key":"carbon%1:27:01::","synset_id":"carbon_paper.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"carbon%1:27:00::","call_ids":["semeval2013.d007.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"carbon%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.00372,"input_uncached_usd":0.00231,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5406745839864016}
{"item_id":"semeval2013.d007.s006.t006","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"},{"index":2,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s006.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":438,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.0037500000000000003,"input_uncached_usd":0.00219,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5784369159955531}
{"item_id":"semeval2013.d007.s007.t003","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.003955,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4274715000065044}
{"item_id":"semeval2013.d007.s007.t004","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"},{"index":2,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.0036650000000000003,"input_uncached_usd":0.0021650000000000003,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6501592920394614}
{"item_id":"semeval2013.d007.s008.t000","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":2,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":3,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":6,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"},{"index":7,"sense_key":"report%1:10:01::","synset_id":"report.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2013.d007.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.005600000000000001,"input_uncached_usd":0.0032600000000000003,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7222094170283526}
{"item_id":"semeval2013.d007.s009.t000","gold_sense_keys":["find%1:09:00::"],"candidates":[{"index":1,"sense_key":"find%1:09:00::","synset_id":"discovery.n.03"},{"index":2,"sense_key":"find%1:04:00::","synset_id":"discovery.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"find%1:09:00::","call_ids":["semeval2013.d007.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"find%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":386,"cached_input_tokens":0,"output_tokens":222,"reasoning_output_tokens":207},"cost":{"currency":"USD","total_usd":0.00859,"input_uncached_usd":0.00193,"input_cached_usd":0.0,"output_usd":0.00666,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.120115999947302}
{"item_id":"semeval2013.d007.s009.t001","gold_sense_keys":["search%1:04:00::","search%1:04:02::"],"candidates":[{"index":1,"sense_key":"search%1:04:02::","synset_id":"search.n.02"},{"index":2,"sense_key":"search%1:09:00::","synset_id":"search.n.04"},{"index":3,"sense_key":"search%1:22:00::","synset_id":"search.n.03"},{"index":4,"sense_key":"search%1:04:01::","synset_id":"search.n.05"},{"index":5,"sense_key":"search%1:04:00::","synset_id":"search.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"search%1:04:00::","call_ids":["semeval2013.d007.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"search%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":279,"reasoning_output_tokens":263},"cost":{"currency":"USD","total_usd":0.01099,"input_uncached_usd":0.0026200000000000004,"input_cached_usd":0.0,"output_usd":0.00837,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.728320749942213}
{"item_id":"semeval2013.d007.s009.t002","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"},{"index":2,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":3,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":4,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":5,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":6,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":7,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":8,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":9,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":10,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":11,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":12,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":13,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":14,"sense_key":"life%1:26:02::","synset_id":"life.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":900,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02034,"input_uncached_usd":0.0045000000000000005,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.592911083018407}
{"item_id":"semeval2013.d007.s009.t003","gold_sense_keys":["earth%1:17:00::"],"candidates":[{"index":1,"sense_key":"earth%1:15:00::","synset_id":"earth.n.04"},{"index":2,"sense_key":"earth%1:06:00::","synset_id":"ground.n.09"},{"index":3,"sense_key":"earth%1:17:01::","synset_id":"land.n.04"},{"index":4,"sense_key":"earth%1:09:00::","synset_id":"worldly_concern.n.01"},{"index":5,"sense_key":"earth%1:27:00::","synset_id":"earth.n.02"},{"index":6,"sense_key":"earth%1:17:00::","synset_id":"earth.n.01"},{"index":7,"sense_key":"earth%1:27:01::","synset_id":"earth.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"earth%1:17:00::","call_ids":["semeval2013.d007.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"earth%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.005175,"input_uncached_usd":0.0033150000000000002,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":64.87448904197663}
{"item_id":"semeval2013.d007.s009.t005","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":3,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":4,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":5,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":6,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":7,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":8,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":9,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":10,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":11,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":12,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":13,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":886,"reasoning_output_tokens":864},"cost":{"currency":"USD","total_usd":0.031075000000000005,"input_uncached_usd":0.004495000000000001,"input_cached_usd":0.0,"output_usd":0.02658,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":18.232196249999106}
{"item_id":"semeval2013.d007.s009.t006","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":3,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":4,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":5,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"},{"index":6,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":7,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":8,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":9,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":10,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":11,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":12,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":13,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":14,"sense_key":"life%1:19:00::","synset_id":"life.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":900,"cached_input_tokens":0,"output_tokens":1011,"reasoning_output_tokens":989},"cost":{"currency":"USD","total_usd":0.03483,"input_uncached_usd":0.0045000000000000005,"input_cached_usd":0.0,"output_usd":0.03033,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":14.39342462492641}
{"item_id":"semeval2013.d007.s010.t000","gold_sense_keys":["study%1:10:00::"],"candidates":[{"index":1,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":2,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":3,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":7,"sense_key":"study%1:10:01::","synset_id":"study.n.10"},{"index":8,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":9,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":10,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2013.d007.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":775,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019715,"input_uncached_usd":0.0038750000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.572991832974367}
{"item_id":"semeval2013.d007.s010.t002","gold_sense_keys":["earth%1:17:00::"],"candidates":[{"index":1,"sense_key":"earth%1:17:01::","synset_id":"land.n.04"},{"index":2,"sense_key":"earth%1:15:00::","synset_id":"earth.n.04"},{"index":3,"sense_key":"earth%1:09:00::","synset_id":"worldly_concern.n.01"},{"index":4,"sense_key":"earth%1:27:01::","synset_id":"earth.n.05"},{"index":5,"sense_key":"earth%1:06:00::","synset_id":"ground.n.09"},{"index":6,"sense_key":"earth%1:17:00::","synset_id":"earth.n.01"},{"index":7,"sense_key":"earth%1:27:00::","synset_id":"earth.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"earth%1:17:00::","call_ids":["semeval2013.d007.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"earth%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.005345000000000001,"input_uncached_usd":0.003455,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4787879579234868}
{"item_id":"semeval2013.d007.s011.t002","gold_sense_keys":["detail%1:09:00::"],"candidates":[{"index":1,"sense_key":"detail%1:09:00::","synset_id":"detail.n.01"},{"index":2,"sense_key":"detail%1:14:01::","synset_id":"detail.n.04"},{"index":3,"sense_key":"detail%1:10:00::","synset_id":"detail.n.03"},{"index":4,"sense_key":"detail%1:14:00::","synset_id":"contingent.n.02"},{"index":5,"sense_key":"detail%1:24:00::","synset_id":"detail.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"detail%1:09:00::","call_ids":["semeval2013.d007.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"detail%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":122},"cost":{"currency":"USD","total_usd":0.006670000000000001,"input_uncached_usd":0.00253,"input_cached_usd":0.0,"output_usd":0.0041400000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0591291659511626}
{"item_id":"semeval2013.d007.s011.t003","gold_sense_keys":["find%1:09:00::"],"candidates":[{"index":1,"sense_key":"find%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"find%1:09:00::","synset_id":"discovery.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"find%1:09:00::","call_ids":["semeval2013.d007.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"find%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":366,"cached_input_tokens":0,"output_tokens":462,"reasoning_output_tokens":446},"cost":{"currency":"USD","total_usd":0.01569,"input_uncached_usd":0.0018300000000000002,"input_cached_usd":0.0,"output_usd":0.01386,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.998484957963228}
{"item_id":"semeval2013.d007.s011.t004","gold_sense_keys":["speculation%1:09:00::"],"candidates":[{"index":1,"sense_key":"speculation%1:09:01::","synset_id":"meditation.n.01"},{"index":2,"sense_key":"speculation%1:21:00::","synset_id":"speculation.n.03"},{"index":3,"sense_key":"speculation%1:09:00::","synset_id":"speculation.n.02"},{"index":4,"sense_key":"speculation%1:10:03::","synset_id":"guess.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"speculation%1:09:00::","call_ids":["semeval2013.d007.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"speculation%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":398,"reasoning_output_tokens":382},"cost":{"currency":"USD","total_usd":0.014565,"input_uncached_usd":0.002625,"input_cached_usd":0.0,"output_usd":0.011940000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.46305387490429}
{"item_id":"semeval2013.d007.s011.t006","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"},{"index":2,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":3,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":4,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":5,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":6,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":7,"sense_key":"report%1:10:01::","synset_id":"report.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2013.d007.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.00589,"input_uncached_usd":0.00319,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.322895959019661}
{"item_id":"semeval2013.d007.s011.t007","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":3,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discovery%1:04:00::","call_ids":["semeval2013.d007.s011.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discovery%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":238,"reasoning_output_tokens":222},"cost":{"currency":"USD","total_usd":0.009355,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.0071400000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.612309124902822}
{"item_id":"semeval2013.d007.s013.t000","gold_sense_keys":["discovery%1:09:00::","discovery%1:10:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"},{"index":2,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":3,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":4,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"discovery%1:10:00::","call_ids":["semeval2013.d007.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"discovery%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018185,"input_uncached_usd":0.0023450000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.395700542023405}
{"item_id":"semeval2013.d007.s013.t001","gold_sense_keys":["environment%1:26:00::"],"candidates":[{"index":1,"sense_key":"environment%1:15:00::","synset_id":"environment.n.02"},{"index":2,"sense_key":"environment%1:26:00::","synset_id":"environment.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"environment%1:26:00::","call_ids":["semeval2013.d007.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"environment%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.014100000000000001,"input_uncached_usd":0.0021000000000000003,"input_cached_usd":0.0,"output_usd":0.012,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.301910208072513}
{"item_id":"semeval2013.d007.s013.t002","gold_sense_keys":["organism%1:03:00::"],"candidates":[{"index":1,"sense_key":"organism%1:03:00::","synset_id":"organism.n.01"},{"index":2,"sense_key":"organism%1:14:00::","synset_id":"organism.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"organism%1:03:00::","call_ids":["semeval2013.d007.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"organism%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":413,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.003895,"input_uncached_usd":0.002065,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6897985000396147}
{"item_id":"semeval2013.d007.s013.t003","gold_sense_keys":["element%1:27:00::"],"candidates":[{"index":1,"sense_key":"element%1:25:00::","synset_id":"element.n.07"},{"index":2,"sense_key":"element%1:06:00::","synset_id":"component.n.03"},{"index":3,"sense_key":"element%1:26:00::","synset_id":"element.n.06"},{"index":4,"sense_key":"element%1:27:01::","synset_id":"element.n.05"},{"index":5,"sense_key":"element%1:15:00::","synset_id":"element.n.04"},{"index":6,"sense_key":"element%1:27:00::","synset_id":"chemical_element.n.01"},{"index":7,"sense_key":"element%1:09:00::","synset_id":"component.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"element%1:27:00::","call_ids":["semeval2013.d007.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"element%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.00494,"input_uncached_usd":0.0032900000000000004,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5107544590719044}
{"item_id":"semeval2013.d007.s013.t005","gold_sense_keys":["set%1:14:00::"],"candidates":[{"index":1,"sense_key":"set%1:28:00::","synset_id":"set.n.08"},{"index":2,"sense_key":"set%1:14:02::","synset_id":"set.n.02"},{"index":3,"sense_key":"set%1:09:00::","synset_id":"set.n.12"},{"index":4,"sense_key":"set%1:11:00::","synset_id":"set.n.11"},{"index":5,"sense_key":"set%1:04:00::","synset_id":"set.n.03"},{"index":6,"sense_key":"set%1:14:01::","synset_id":"set.n.05"},{"index":7,"sense_key":"set%1:06:01::","synset_id":"set.n.13"},{"index":8,"sense_key":"set%1:04:01::","synset_id":"set.n.07"},{"index":9,"sense_key":"set%1:22:00::","synset_id":"hardening.n.02"},{"index":10,"sense_key":"set%1:14:00::","synset_id":"set.n.01"},{"index":11,"sense_key":"set%1:09:01::","synset_id":"bent.n.01"},{"index":12,"sense_key":"set%1:06:00::","synset_id":"stage_set.n.01"},{"index":13,"sense_key":"set%1:18:00::","synset_id":"set.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"set%1:14:00::","call_ids":["semeval2013.d007.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"set%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":897,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.007065,"input_uncached_usd":0.004485,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1228499580174685}
{"item_id":"semeval2013.d007.s013.t006","gold_sense_keys":["element%1:27:00::"],"candidates":[{"index":1,"sense_key":"element%1:26:00::","synset_id":"element.n.06"},{"index":2,"sense_key":"element%1:27:01::","synset_id":"element.n.05"},{"index":3,"sense_key":"element%1:06:00::","synset_id":"component.n.03"},{"index":4,"sense_key":"element%1:15:00::","synset_id":"element.n.04"},{"index":5,"sense_key":"element%1:27:00::","synset_id":"chemical_element.n.01"},{"index":6,"sense_key":"element%1:25:00::","synset_id":"element.n.07"},{"index":7,"sense_key":"element%1:09:00::","synset_id":"component.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"element%1:27:00::","call_ids":["semeval2013.d007.s013.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"element%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.00506,"input_uncached_usd":0.0032900000000000004,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6230012499727309}
{"item_id":"semeval2013.d007.s013.t007","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":2,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":3,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":4,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":5,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":6,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":7,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":8,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":9,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":10,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":11,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":12,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":13,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":907,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.035914999999999996,"input_uncached_usd":0.004535,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":16.193444584030658}
{"item_id":"semeval2013.d007.s013.t009","gold_sense_keys":["space%1:15:01::"],"candidates":[{"index":1,"sense_key":"space%1:25:00::","synset_id":"space.n.02"},{"index":2,"sense_key":"space%1:15:00::","synset_id":"space.n.03"},{"index":3,"sense_key":"space%1:03:00::","synset_id":"space.n.01"},{"index":4,"sense_key":"space%1:15:01::","synset_id":"outer_space.n.01"},{"index":5,"sense_key":"space%1:10:02::","synset_id":"space.n.08"},{"index":6,"sense_key":"space%1:10:01::","synset_id":"space.n.07"},{"index":7,"sense_key":"space%1:28:00::","synset_id":"distance.n.05"},{"index":8,"sense_key":"space%1:06:00::","synset_id":"quad.n.03"},{"index":9,"sense_key":"space%1:10:00::","synset_id":"space.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"space%1:15:01::","call_ids":["semeval2013.d007.s013.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"space%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":243,"reasoning_output_tokens":227},"cost":{"currency":"USD","total_usd":0.011035000000000001,"input_uncached_usd":0.0037450000000000005,"input_cached_usd":0.0,"output_usd":0.0072900000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.3738431660458446}
{"item_id":"semeval2013.d007.s013.t010","gold_sense_keys":["science%1:09:00::"],"candidates":[{"index":1,"sense_key":"science%1:09:02::","synset_id":"skill.n.02"},{"index":2,"sense_key":"science%1:09:00::","synset_id":"science.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"science%1:09:00::","call_ids":["semeval2013.d007.s013.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"science%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.004240000000000001,"input_uncached_usd":0.0021100000000000003,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1075951249804348}
{"item_id":"semeval2013.d007.s014.t000","gold_sense_keys":["part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":2,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":3,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":6,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":7,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":8,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":9,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":10,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":11,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"},{"index":12,"sense_key":"part%1:09:01::","synset_id":"character.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"part%1:24:00::","call_ids":["semeval2013.d007.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"part%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":932,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":117},"cost":{"currency":"USD","total_usd":0.008650000000000001,"input_uncached_usd":0.00466,"input_cached_usd":0.0,"output_usd":0.0039900000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0847038329811767}
{"item_id":"semeval2013.d007.s014.t002","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d007.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":394,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.00344,"input_uncached_usd":0.00197,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3388492499943823}
{"item_id":"semeval2013.d007.s015.t000","gold_sense_keys":["work%1:06:00::"],"candidates":[{"index":1,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":2,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":3,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":4,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":5,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":6,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":7,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"work%1:06:00::","call_ids":["semeval2013.d007.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"work%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":674,"cached_input_tokens":0,"output_tokens":458,"reasoning_output_tokens":442},"cost":{"currency":"USD","total_usd":0.01711,"input_uncached_usd":0.00337,"input_cached_usd":0.0,"output_usd":0.01374,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.734542541089468}
{"item_id":"semeval2013.d007.s015.t001","gold_sense_keys":["substitution%1:11:00::"],"candidates":[{"index":1,"sense_key":"substitution%1:11:00::","synset_id":"substitution.n.01"},{"index":2,"sense_key":"substitution%1:04:00::","synset_id":"substitution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"substitution%1:11:00::","call_ids":["semeval2013.d007.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"substitution%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":423,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":141},"cost":{"currency":"USD","total_usd":0.006825000000000001,"input_uncached_usd":0.002115,"input_cached_usd":0.0,"output_usd":0.00471,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3224470829591155}
{"item_id":"semeval2013.d007.s015.t002","gold_sense_keys":["element%1:27:00::"],"candidates":[{"index":1,"sense_key":"element%1:27:01::","synset_id":"element.n.05"},{"index":2,"sense_key":"element%1:09:00::","synset_id":"component.n.01"},{"index":3,"sense_key":"element%1:25:00::","synset_id":"element.n.07"},{"index":4,"sense_key":"element%1:27:00::","synset_id":"chemical_element.n.01"},{"index":5,"sense_key":"element%1:26:00::","synset_id":"element.n.06"},{"index":6,"sense_key":"element%1:15:00::","synset_id":"element.n.04"},{"index":7,"sense_key":"element%1:06:00::","synset_id":"component.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"element%1:27:00::","call_ids":["semeval2013.d007.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"element%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":632,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.005050000000000001,"input_uncached_usd":0.00316,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9052051670150831}
{"item_id":"semeval2013.d007.s015.t004","gold_sense_keys":["structure%1:07:00::"],"candidates":[{"index":1,"sense_key":"structure%1:14:00::","synset_id":"social_organization.n.01"},{"index":2,"sense_key":"structure%1:08:00::","synset_id":"structure.n.04"},{"index":3,"sense_key":"structure%1:06:00::","synset_id":"structure.n.01"},{"index":4,"sense_key":"structure%1:07:00::","synset_id":"structure.n.02"},{"index":5,"sense_key":"structure%1:09:00::","synset_id":"structure.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"structure%1:07:00::","call_ids":["semeval2013.d007.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"structure%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":161,"reasoning_output_tokens":145},"cost":{"currency":"USD","total_usd":0.0077,"input_uncached_usd":0.00287,"input_cached_usd":0.0,"output_usd":0.00483,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.786026166984811}
{"item_id":"semeval2013.d007.s016.t000","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":2,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":3,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"discovery%1:04:00::","call_ids":["semeval2013.d007.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"discovery%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":362,"reasoning_output_tokens":346},"cost":{"currency":"USD","total_usd":0.01316,"input_uncached_usd":0.0023000000000000004,"input_cached_usd":0.0,"output_usd":0.01086,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.787885624915361}
{"item_id":"semeval2013.d007.s017.t003","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":2,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":3,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:10:01::","synset_id":"study.n.10"},{"index":9,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":10,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2013.d007.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":247,"reasoning_output_tokens":231},"cost":{"currency":"USD","total_usd":0.011085000000000001,"input_uncached_usd":0.0036750000000000003,"input_cached_usd":0.0,"output_usd":0.00741,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.448777707992122}
{"item_id":"semeval2013.d007.s017.t004","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"},{"index":2,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s017.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":458,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.00457,"input_uncached_usd":0.0022900000000000004,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7957233749330044}
{"item_id":"semeval2013.d007.s017.t005","gold_sense_keys":["level%1:07:00::"],"candidates":[{"index":1,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":2,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":3,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":4,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":5,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":6,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"},{"index":7,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":8,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"level%1:07:00::","call_ids":["semeval2013.d007.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"level%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":683,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.005755,"input_uncached_usd":0.003415,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8670154579449445}
{"item_id":"semeval2013.d007.s018.t000","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.004505,"input_uncached_usd":0.0023750000000000004,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7067922920687124}
{"item_id":"semeval2013.d007.s018.t001","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"},{"index":2,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.003765,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9446860840544105}
{"item_id":"semeval2013.d007.s019.t000","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"},{"index":2,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.00408,"input_uncached_usd":0.0021000000000000003,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8222721660276875}
{"item_id":"semeval2013.d007.s019.t002","gold_sense_keys":["behavior%1:26:00::"],"candidates":[{"index":1,"sense_key":"behavior%1:26:00::","synset_id":"behavior.n.02"},{"index":2,"sense_key":"behavior%1:04:01::","synset_id":"behavior.n.04"},{"index":3,"sense_key":"behavior%1:07:00::","synset_id":"demeanor.n.01"},{"index":4,"sense_key":"behavior%1:04:00::","synset_id":"behavior.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"behavior%1:26:00::","call_ids":["semeval2013.d007.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"behavior%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.003885,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5353026250377297}
{"item_id":"semeval2013.d007.s019.t003","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"},{"index":2,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":393,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.003585,"input_uncached_usd":0.001965,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.113649541977793}
{"item_id":"semeval2013.d007.s020.t000","gold_sense_keys":["result%1:19:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"},{"index":4,"sense_key":"result%1:11:00::","synset_id":"result.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"result%1:19:00::","call_ids":["semeval2013.d007.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"result%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.004785,"input_uncached_usd":0.002595,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7913669999688864}
{"item_id":"semeval2013.d007.s020.t001","gold_sense_keys":["organism%1:03:00::"],"candidates":[{"index":1,"sense_key":"organism%1:03:00::","synset_id":"organism.n.01"},{"index":2,"sense_key":"organism%1:14:00::","synset_id":"organism.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"organism%1:03:00::","call_ids":["semeval2013.d007.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"organism%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":384,"cached_input_tokens":0,"output_tokens":42,"reasoning_output_tokens":27},"cost":{"currency":"USD","total_usd":0.0031800000000000005,"input_uncached_usd":0.0019200000000000003,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3729057499440387}
{"item_id":"semeval2013.d007.s020.t002","gold_sense_keys":["hard_time%1:26:00::"],"candidates":[{"index":1,"sense_key":"hard_time%1:26:00::","synset_id":"hard_time.n.01"},{"index":2,"sense_key":"hard_time%1:28:00::","synset_id":"hard_time.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hard_time%1:26:00::","call_ids":["semeval2013.d007.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hard_time%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.003915,"input_uncached_usd":0.001935,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8875304580433294}
{"item_id":"semeval2013.d007.s020.t003","gold_sense_keys":["element%1:27:00::"],"candidates":[{"index":1,"sense_key":"element%1:06:00::","synset_id":"component.n.03"},{"index":2,"sense_key":"element%1:26:00::","synset_id":"element.n.06"},{"index":3,"sense_key":"element%1:27:00::","synset_id":"chemical_element.n.01"},{"index":4,"sense_key":"element%1:25:00::","synset_id":"element.n.07"},{"index":5,"sense_key":"element%1:27:01::","synset_id":"element.n.05"},{"index":6,"sense_key":"element%1:09:00::","synset_id":"component.n.01"},{"index":7,"sense_key":"element%1:15:00::","synset_id":"element.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"element%1:27:00::","call_ids":["semeval2013.d007.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"element%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.004465,"input_uncached_usd":0.0031450000000000002,"input_cached_usd":0.0,"output_usd":0.00132,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2874870829982683}
{"item_id":"semeval2013.d007.s021.t000","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.003940000000000001,"input_uncached_usd":0.0022600000000000003,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2277481670025736}
{"item_id":"semeval2013.d007.s021.t001","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"},{"index":2,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s021.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":41,"reasoning_output_tokens":26},"cost":{"currency":"USD","total_usd":0.0033500000000000005,"input_uncached_usd":0.0021200000000000004,"input_cached_usd":0.0,"output_usd":0.00123,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.563306082971394}
{"item_id":"semeval2013.d007.s022.t002","gold_sense_keys":["concentration%1:07:02::"],"candidates":[{"index":1,"sense_key":"concentration%1:04:01::","synset_id":"concentration.n.03"},{"index":2,"sense_key":"concentration%1:07:03::","synset_id":"concentration.n.02"},{"index":3,"sense_key":"concentration%1:07:01::","synset_id":"assiduity.n.01"},{"index":4,"sense_key":"concentration%1:04:02::","synset_id":"concentration.n.06"},{"index":5,"sense_key":"concentration%1:09:00::","synset_id":"concentration.n.05"},{"index":6,"sense_key":"concentration%1:11:00::","synset_id":"concentration.n.04"},{"index":7,"sense_key":"concentration%1:07:02::","synset_id":"concentration.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"concentration%1:07:02::","call_ids":["semeval2013.d007.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"concentration%1:07:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0048400000000000006,"input_uncached_usd":0.0029200000000000003,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1239728749496862}
{"item_id":"semeval2013.d007.s022.t003","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"},{"index":2,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":123},"cost":{"currency":"USD","total_usd":0.006615000000000001,"input_uncached_usd":0.002475,"input_cached_usd":0.0,"output_usd":0.0041400000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.229208957985975}
{"item_id":"semeval2013.d007.s023.t001","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"},{"index":2,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.003855,"input_uncached_usd":0.002415,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5855571670690551}
{"item_id":"semeval2013.d007.s023.t002","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.004475,"input_uncached_usd":0.0025550000000000004,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.15205545804929}
{"item_id":"semeval2013.d007.s023.t003","gold_sense_keys":["possibility%1:26:00::"],"candidates":[{"index":1,"sense_key":"possibility%1:09:00::","synset_id":"hypothesis.n.02"},{"index":2,"sense_key":"possibility%1:09:03::","synset_id":"possibility.n.01"},{"index":3,"sense_key":"possibility%1:26:00::","synset_id":"possibility.n.02"},{"index":4,"sense_key":"possibility%1:09:01::","synset_id":"possibility.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"possibility%1:26:00::","call_ids":["semeval2013.d007.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"possibility%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.005325,"input_uncached_usd":0.0028350000000000003,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6794642920140177}
{"item_id":"semeval2013.d007.s023.t005","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"},{"index":2,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.004385,"input_uncached_usd":0.0025550000000000004,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9222925000358373}
{"item_id":"semeval2013.d007.s023.t006","gold_sense_keys":["earth%1:17:00::"],"candidates":[{"index":1,"sense_key":"earth%1:17:01::","synset_id":"land.n.04"},{"index":2,"sense_key":"earth%1:17:00::","synset_id":"earth.n.01"},{"index":3,"sense_key":"earth%1:06:00::","synset_id":"ground.n.09"},{"index":4,"sense_key":"earth%1:27:01::","synset_id":"earth.n.05"},{"index":5,"sense_key":"earth%1:09:00::","synset_id":"worldly_concern.n.01"},{"index":6,"sense_key":"earth%1:15:00::","synset_id":"earth.n.04"},{"index":7,"sense_key":"earth%1:27:00::","synset_id":"earth.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"earth%1:17:00::","call_ids":["semeval2013.d007.s023.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"earth%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":706,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.0053,"input_uncached_usd":0.00353,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6366742920363322}
{"item_id":"semeval2013.d007.s023.t007","gold_sense_keys":["planet%1:17:00::","planet%1:17:01::"],"candidates":[{"index":1,"sense_key":"planet%1:17:01::","synset_id":"planet.n.03"},{"index":2,"sense_key":"planet%1:18:00::","synset_id":"satellite.n.02"},{"index":3,"sense_key":"planet%1:17:00::","synset_id":"planet.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"planet%1:17:01::","call_ids":["semeval2013.d007.s023.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"planet%1:17:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.0050750000000000005,"input_uncached_usd":0.002645,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2512492920504883}
{"item_id":"semeval2013.d007.s023.t008","gold_sense_keys":["moon%1:17:00::"],"candidates":[{"index":1,"sense_key":"moon%1:17:02::","synset_id":"moon.n.02"},{"index":2,"sense_key":"moon%1:17:01::","synset_id":"moon.n.01"},{"index":3,"sense_key":"moon%1:28:00::","synset_id":"lunar_month.n.01"},{"index":4,"sense_key":"moon%1:18:00::","synset_id":"moon.n.05"},{"index":5,"sense_key":"moon%1:17:00::","synset_id":"moon.n.06"},{"index":6,"sense_key":"moon%1:19:00::","synset_id":"moonlight.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"moon%1:17:00::","call_ids":["semeval2013.d007.s023.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"moon%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.005220000000000001,"input_uncached_usd":0.0032700000000000003,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6686742500169203}
{"item_id":"semeval2013.d007.s023.t009","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"},{"index":2,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s023.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.0044150000000000005,"input_uncached_usd":0.0025550000000000004,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6980324999894947}
{"item_id":"semeval2013.d007.s024.t002","gold_sense_keys":["idea%1:09:00::"],"candidates":[{"index":1,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"},{"index":2,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"},{"index":3,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"},{"index":4,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"},{"index":5,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"idea%1:09:00::","call_ids":["semeval2013.d007.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"idea%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.005645000000000001,"input_uncached_usd":0.003095,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0698239589110017}
{"item_id":"semeval2013.d007.s024.t003","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"},{"index":2,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s024.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.004205,"input_uncached_usd":0.0025550000000000004,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":70.76196899998467}
{"item_id":"semeval2013.d007.s024.t004","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"},{"index":2,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.00383,"input_uncached_usd":0.0024200000000000003,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5575129579519853}
{"item_id":"semeval2013.d007.s024.t005","gold_sense_keys":["example%1:11:00::"],"candidates":[{"index":1,"sense_key":"example%1:09:00::","synset_id":"example.n.01"},{"index":2,"sense_key":"example%1:04:00::","synset_id":"exercise.n.04"},{"index":3,"sense_key":"example%1:11:00::","synset_id":"case.n.01"},{"index":4,"sense_key":"example%1:10:00::","synset_id":"example.n.04"},{"index":5,"sense_key":"example%1:09:02::","synset_id":"model.n.07"},{"index":6,"sense_key":"example%1:09:01::","synset_id":"exemplar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"example%1:11:00::","call_ids":["semeval2013.d007.s024.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"example%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":650,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":115},"cost":{"currency":"USD","total_usd":0.00715,"input_uncached_usd":0.0032500000000000003,"input_cached_usd":0.0,"output_usd":0.0039000000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1474522079806775}
{"item_id":"semeval2013.d007.s025.t000","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"},{"index":2,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.0047550000000000005,"input_uncached_usd":0.0026550000000000002,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0045368330320343}
{"item_id":"semeval2013.d007.s025.t001","gold_sense_keys":["times%1:28:01::"],"candidates":[{"index":1,"sense_key":"prison_term%1:28:00::","synset_id":"prison_term.n.01"},{"index":2,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":3,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":4,"sense_key":"meter%1:07:00::","synset_id":"meter.n.04"},{"index":5,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":6,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":7,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":8,"sense_key":"fourth_dimension%1:28:00::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"times%1:28:01::","synset_id":"times.n.01"},{"index":10,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":11,"sense_key":"clock_time%1:28:00::","synset_id":"clock_time.n.01"},{"index":12,"sense_key":"times%1:04:00::","synset_id":"multiplication.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"times%1:28:01::","call_ids":["semeval2013.d007.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"times%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":926,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":118},"cost":{"currency":"USD","total_usd":0.008650000000000001,"input_uncached_usd":0.0046300000000000004,"input_cached_usd":0.0,"output_usd":0.00402,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.580457958043553}
{"item_id":"semeval2013.d007.s025.t002","gold_sense_keys":["earth%1:17:00::"],"candidates":[{"index":1,"sense_key":"earth%1:15:00::","synset_id":"earth.n.04"},{"index":2,"sense_key":"earth%1:09:00::","synset_id":"worldly_concern.n.01"},{"index":3,"sense_key":"earth%1:27:01::","synset_id":"earth.n.05"},{"index":4,"sense_key":"earth%1:06:00::","synset_id":"ground.n.09"},{"index":5,"sense_key":"earth%1:17:01::","synset_id":"land.n.04"},{"index":6,"sense_key":"earth%1:17:00::","synset_id":"earth.n.01"},{"index":7,"sense_key":"earth%1:27:00::","synset_id":"earth.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"earth%1:17:00::","call_ids":["semeval2013.d007.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"earth%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.0054800000000000005,"input_uncached_usd":0.0036200000000000004,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8076744159916416}
{"item_id":"semeval2013.d007.s026.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":2,"sense_key":"way%1:21:00::","synset_id":"way.n.12"},{"index":3,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":4,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":5,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":6,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":7,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":8,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":9,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":10,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":11,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":12,"sense_key":"way%1:06:00::","synset_id":"way.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"way%1:04:01::","call_ids":["semeval2013.d007.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":868,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02018,"input_uncached_usd":0.00434,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.484310416970402}
{"item_id":"semeval2013.d007.s026.t001","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":3,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":4,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":5,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":6,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":7,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":8,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":9,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":10,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":14,"sense_key":"life%1:28:02::","synset_id":"life.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":964,"cached_input_tokens":0,"output_tokens":323,"reasoning_output_tokens":307},"cost":{"currency":"USD","total_usd":0.014509999999999999,"input_uncached_usd":0.0048200000000000005,"input_cached_usd":0.0,"output_usd":0.00969,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.0578141659498215}
{"item_id":"semeval2013.d007.s026.t002","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":2,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":3,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":4,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"},{"index":5,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":6,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":7,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":8,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":9,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":10,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":11,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":12,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":13,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":14,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":965,"cached_input_tokens":0,"output_tokens":308,"reasoning_output_tokens":292},"cost":{"currency":"USD","total_usd":0.014065000000000001,"input_uncached_usd":0.004825,"input_cached_usd":0.0,"output_usd":0.00924,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.912734957993962}
{"item_id":"semeval2013.d007.s026.t003","gold_sense_keys":["earth%1:17:00::"],"candidates":[{"index":1,"sense_key":"earth%1:06:00::","synset_id":"ground.n.09"},{"index":2,"sense_key":"earth%1:15:00::","synset_id":"earth.n.04"},{"index":3,"sense_key":"earth%1:27:01::","synset_id":"earth.n.05"},{"index":4,"sense_key":"earth%1:17:01::","synset_id":"land.n.04"},{"index":5,"sense_key":"earth%1:17:00::","synset_id":"earth.n.01"},{"index":6,"sense_key":"earth%1:27:00::","synset_id":"earth.n.02"},{"index":7,"sense_key":"earth%1:09:00::","synset_id":"worldly_concern.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"earth%1:17:00::","call_ids":["semeval2013.d007.s026.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"earth%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.005050000000000001,"input_uncached_usd":0.0036400000000000004,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":56.13034545897972}
{"item_id":"semeval2013.d007.s026.t004","gold_sense_keys":["part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":2,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":3,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":4,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":5,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":6,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":7,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":8,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":9,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":10,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":11,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"},{"index":12,"sense_key":"part%1:15:00::","synset_id":"region.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"part%1:24:00::","call_ids":["semeval2013.d007.s026.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"part%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":990,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":108},"cost":{"currency":"USD","total_usd":0.00867,"input_uncached_usd":0.00495,"input_cached_usd":0.0,"output_usd":0.00372,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.619905999978073}
{"item_id":"semeval2013.d007.s026.t006","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d007.s026.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.0037600000000000003,"input_uncached_usd":0.0022600000000000003,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.912070915917866}
{"item_id":"semeval2013.d008.s000.t000","gold_sense_keys":["routine%1:04:00::"],"candidates":[{"index":1,"sense_key":"routine%1:10:01::","synset_id":"act.n.04"},{"index":2,"sense_key":"routine%1:10:00::","synset_id":"routine.n.03"},{"index":3,"sense_key":"routine%1:04:00::","synset_id":"routine.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"routine%1:04:00::","call_ids":["semeval2013.d008.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"routine%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":294,"cached_input_tokens":0,"output_tokens":184,"reasoning_output_tokens":169},"cost":{"currency":"USD","total_usd":0.00699,"input_uncached_usd":0.0014700000000000002,"input_cached_usd":0.0,"output_usd":0.00552,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.189476125058718}
{"item_id":"semeval2013.d008.s001.t000","gold_sense_keys":["support%1:04:00::","support%1:09:00::"],"candidates":[{"index":1,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":2,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":3,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":4,"sense_key":"support%1:21:01::","synset_id":"support.n.11"},{"index":5,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":6,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":7,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":8,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":9,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":10,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":11,"sense_key":"support%1:06:00::","synset_id":"support.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"support%1:09:00::","call_ids":["semeval2013.d008.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"support%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":371,"reasoning_output_tokens":355},"cost":{"currency":"USD","total_usd":0.014584999999999999,"input_uncached_usd":0.003455,"input_cached_usd":0.0,"output_usd":0.011130000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.580758500029333}
{"item_id":"semeval2013.d008.s001.t003","gold_sense_keys":["backing%1:04:00::"],"candidates":[{"index":1,"sense_key":"backing%1:04:00::","synset_id":"backing.n.01"},{"index":2,"sense_key":"backing%1:06:00::","synset_id":"backing.n.02"},{"index":3,"sense_key":"backing%1:21:00::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"backing%1:04:00::","call_ids":["semeval2013.d008.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"backing%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":308,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.00466,"input_uncached_usd":0.0015400000000000001,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4227495830273256}
{"item_id":"semeval2013.d008.s002.t000","gold_sense_keys":["crime%1:04:00::"],"candidates":[{"index":1,"sense_key":"crime%1:04:00::","synset_id":"crime.n.01"},{"index":2,"sense_key":"crime%1:04:01::","synset_id":"crime.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crime%1:04:00::","call_ids":["semeval2013.d008.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crime%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":284,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.00379,"input_uncached_usd":0.00142,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.554394750040956}
{"item_id":"semeval2013.d008.s002.t001","gold_sense_keys":["worry%1:09:00::"],"candidates":[{"index":1,"sense_key":"worry%1:09:00::","synset_id":"concern.n.04"},{"index":2,"sense_key":"worry%1:12:00::","synset_id":"worry.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worry%1:09:00::","call_ids":["semeval2013.d008.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worry%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":295,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.0031850000000000003,"input_uncached_usd":0.0014750000000000002,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0243915000464767}
{"item_id":"semeval2013.d008.s003.t000","gold_sense_keys":["brazil%1:15:00::"],"candidates":[{"index":1,"sense_key":"brazil%1:13:00::","synset_id":"brazil_nut.n.02"},{"index":2,"sense_key":"brazil%1:15:00::","synset_id":"brazil.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"brazil%1:15:00::","call_ids":["semeval2013.d008.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"brazil%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":312,"cached_input_tokens":0,"output_tokens":39,"reasoning_output_tokens":24},"cost":{"currency":"USD","total_usd":0.0027300000000000002,"input_uncached_usd":0.0015600000000000002,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.726229417021386}
{"item_id":"semeval2013.d008.s003.t001","gold_sense_keys":["united_states%1:15:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"united_states%1:15:00::","call_ids":["semeval2013.d008.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"united_states%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":343,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":106},"cost":{"currency":"USD","total_usd":0.0053750000000000004,"input_uncached_usd":0.0017150000000000002,"input_cached_usd":0.0,"output_usd":0.00366,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4342924170196056}
{"item_id":"semeval2013.d008.s004.t000","gold_sense_keys":["finding%1:17:00::"],"candidates":[{"index":1,"sense_key":"finding%1:04:01::","synset_id":"finding.n.02"},{"index":2,"sense_key":"finding%1:17:00::","synset_id":"finding.n.03"},{"index":3,"sense_key":"finding%1:04:00::","synset_id":"determination.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"finding%1:17:00::","call_ids":["semeval2013.d008.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"finding%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":359,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":101},"cost":{"currency":"USD","total_usd":0.005305000000000001,"input_uncached_usd":0.0017950000000000002,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.732752042007633}
{"item_id":"semeval2013.d008.s004.t001","gold_sense_keys":["poll%1:09:00::"],"candidates":[{"index":1,"sense_key":"poll%1:09:00::","synset_id":"poll.n.01"},{"index":2,"sense_key":"poll%1:08:02::","synset_id":"poll.n.03"},{"index":3,"sense_key":"poll%1:05:00::","synset_id":"poll.n.04"},{"index":4,"sense_key":"poll%1:04:00::","synset_id":"poll.n.05"},{"index":5,"sense_key":"poll%1:08:01::","synset_id":"pate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"poll%1:09:00::","call_ids":["semeval2013.d008.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"poll%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.003685,"input_uncached_usd":0.0019450000000000001,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.21122558391653}
{"item_id":"semeval2013.d008.s004.t002","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":2,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":3,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":4,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"country%1:15:00::","call_ids":["semeval2013.d008.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"country%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":270,"reasoning_output_tokens":254},"cost":{"currency":"USD","total_usd":0.01055,"input_uncached_usd":0.0024500000000000004,"input_cached_usd":0.0,"output_usd":0.0081,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.768012917018496}
{"item_id":"semeval2013.d008.s005.t000","gold_sense_keys":["poll%1:09:00::"],"candidates":[{"index":1,"sense_key":"poll%1:09:00::","synset_id":"poll.n.01"},{"index":2,"sense_key":"poll%1:08:01::","synset_id":"pate.n.02"},{"index":3,"sense_key":"poll%1:04:00::","synset_id":"poll.n.05"},{"index":4,"sense_key":"poll%1:08:02::","synset_id":"poll.n.03"},{"index":5,"sense_key":"poll%1:05:00::","synset_id":"poll.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"poll%1:09:00::","call_ids":["semeval2013.d008.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"poll%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.0035450000000000004,"input_uncached_usd":0.0020450000000000004,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5297137919114903}
{"item_id":"semeval2013.d008.s005.t001","gold_sense_keys":["change%1:19:00::","change%1:24:00::"],"candidates":[{"index":1,"sense_key":"change%1:06:01::","synset_id":"change.n.06"},{"index":2,"sense_key":"change%1:21:02::","synset_id":"change.n.05"},{"index":3,"sense_key":"change%1:19:00::","synset_id":"change.n.04"},{"index":4,"sense_key":"change%1:24:00::","synset_id":"change.n.02"},{"index":5,"sense_key":"change%1:11:00::","synset_id":"change.n.01"},{"index":6,"sense_key":"change%1:04:00::","synset_id":"change.n.03"},{"index":7,"sense_key":"change%1:07:00::","synset_id":"variety.n.06"},{"index":8,"sense_key":"change%1:21:01::","synset_id":"change.n.08"},{"index":9,"sense_key":"change%1:21:03::","synset_id":"change.n.09"},{"index":10,"sense_key":"change%1:06:00::","synset_id":"change.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"change%1:24:00::","call_ids":["semeval2013.d008.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"change%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1141},"cost":{"currency":"USD","total_usd":0.03851,"input_uncached_usd":0.0034400000000000003,"input_cached_usd":0.0,"output_usd":0.035070000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":21.36319479206577}
{"item_id":"semeval2013.d008.s005.t002","gold_sense_keys":["attitude%1:09:00::"],"candidates":[{"index":1,"sense_key":"attitude%1:07:01::","synset_id":"position.n.04"},{"index":2,"sense_key":"attitude%1:24:00::","synset_id":"attitude.n.04"},{"index":3,"sense_key":"attitude%1:09:00::","synset_id":"attitude.n.01"},{"index":4,"sense_key":"attitude%1:04:00::","synset_id":"attitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"attitude%1:09:00::","call_ids":["semeval2013.d008.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"attitude%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.0037600000000000003,"input_uncached_usd":0.0021100000000000003,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3840717499842867}
{"item_id":"semeval2013.d008.s006.t000","gold_sense_keys":["thing%1:09:02::"],"candidates":[{"index":1,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":2,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":3,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":6,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":7,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":8,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":9,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":12,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"thing%1:07:00::","call_ids":["semeval2013.d008.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"thing%1:07:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":664,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01916,"input_uncached_usd":0.0033200000000000005,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.498071208014153}
{"item_id":"semeval2013.d008.s006.t001","gold_sense_keys":["result%1:11:00::"],"candidates":[{"index":1,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":2,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"},{"index":3,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":4,"sense_key":"result%1:11:00::","synset_id":"result.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"result%1:11:00::","call_ids":["semeval2013.d008.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"result%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":159,"reasoning_output_tokens":143},"cost":{"currency":"USD","total_usd":0.007155000000000002,"input_uncached_usd":0.0023850000000000004,"input_cached_usd":0.0,"output_usd":0.00477,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2099259169772267}
{"item_id":"semeval2013.d008.s006.t002","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:14:00::","synset_id":"class.n.06"},{"index":4,"sense_key":"year%1:28:00::","synset_id":"year.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":95},"cost":{"currency":"USD","total_usd":0.0056500000000000005,"input_uncached_usd":0.00232,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7889201670186594}
{"item_id":"semeval2013.d008.s006.t003","gold_sense_keys":["poll%1:09:00::"],"candidates":[{"index":1,"sense_key":"poll%1:05:00::","synset_id":"poll.n.04"},{"index":2,"sense_key":"poll%1:08:01::","synset_id":"pate.n.02"},{"index":3,"sense_key":"poll%1:04:00::","synset_id":"poll.n.05"},{"index":4,"sense_key":"poll%1:09:00::","synset_id":"poll.n.01"},{"index":5,"sense_key":"poll%1:08:02::","synset_id":"poll.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"poll%1:09:00::","call_ids":["semeval2013.d008.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"poll%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.00393,"input_uncached_usd":0.0020700000000000002,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7322611659765244}
{"item_id":"semeval2013.d008.s007.t001","gold_sense_keys":["mood%1:12:00::","mood%1:26:00::"],"candidates":[{"index":1,"sense_key":"mood%1:12:00::","synset_id":"temper.n.02"},{"index":2,"sense_key":"mood%1:26:00::","synset_id":"climate.n.02"},{"index":3,"sense_key":"mood%1:24:00::","synset_id":"mood.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mood%1:26:00::","call_ids":["semeval2013.d008.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mood%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":405,"cached_input_tokens":0,"output_tokens":215,"reasoning_output_tokens":200},"cost":{"currency":"USD","total_usd":0.008475,"input_uncached_usd":0.0020250000000000003,"input_cached_usd":0.0,"output_usd":0.00645,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.361550083965994}
{"item_id":"semeval2013.d008.s008.t000","gold_sense_keys":["second%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"second%5:00:00:ordinal:00","synset_id":"second.s.01"},{"index":2,"sense_key":"second%3:00:00::","synset_id":"second.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%5:00:00:ordinal:00","call_ids":["semeval2013.d008.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":365,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.003235,"input_uncached_usd":0.0018250000000000002,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8016138749662787}
{"item_id":"semeval2013.d008.s008.t001","gold_sense_keys":["stability%1:07:01::"],"candidates":[{"index":1,"sense_key":"stability%1:07:01::","synset_id":"constancy.n.01"},{"index":2,"sense_key":"stability%1:07:00::","synset_id":"stability.n.01"},{"index":3,"sense_key":"stability%1:26:00::","synset_id":"stability.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stability%1:07:01::","call_ids":["semeval2013.d008.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stability%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":104},"cost":{"currency":"USD","total_usd":0.005515,"input_uncached_usd":0.0019150000000000003,"input_cached_usd":0.0,"output_usd":0.0036,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7101268749684095}
{"item_id":"semeval2013.d008.s008.t002","gold_sense_keys":["attitude%1:09:00::"],"candidates":[{"index":1,"sense_key":"attitude%1:24:00::","synset_id":"attitude.n.04"},{"index":2,"sense_key":"attitude%1:09:00::","synset_id":"attitude.n.01"},{"index":3,"sense_key":"attitude%1:04:00::","synset_id":"attitude.n.03"},{"index":4,"sense_key":"attitude%1:07:01::","synset_id":"position.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"attitude%1:09:00::","call_ids":["semeval2013.d008.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"attitude%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":438,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.00378,"input_uncached_usd":0.00219,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8385974579723552}
{"item_id":"semeval2013.d008.s008.t004","gold_sense_keys":["institution%1:09:00::"],"candidates":[{"index":1,"sense_key":"institution%1:04:00::","synset_id":"initiation.n.02"},{"index":2,"sense_key":"institution%1:09:00::","synset_id":"institution.n.03"},{"index":3,"sense_key":"institution%1:06:00::","synset_id":"institution.n.02"},{"index":4,"sense_key":"institution%1:06:01::","synset_id":"mental_hospital.n.01"},{"index":5,"sense_key":"institution%1:14:00::","synset_id":"institution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"institution%1:14:00::","call_ids":["semeval2013.d008.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"institution%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":274,"reasoning_output_tokens":258},"cost":{"currency":"USD","total_usd":0.010725,"input_uncached_usd":0.0025050000000000003,"input_cached_usd":0.0,"output_usd":0.00822,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.817827166989446}
{"item_id":"semeval2013.d008.s009.t000","gold_sense_keys":["support%1:04:00::","support%1:09:00::"],"candidates":[{"index":1,"sense_key":"support%1:21:01::","synset_id":"support.n.11"},{"index":2,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":3,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":4,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":5,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":6,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":7,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":10,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":11,"sense_key":"support%1:09:00::","synset_id":"support.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"support%1:09:00::","call_ids":["semeval2013.d008.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"support%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":816,"cached_input_tokens":0,"output_tokens":999,"reasoning_output_tokens":977},"cost":{"currency":"USD","total_usd":0.03405,"input_uncached_usd":0.00408,"input_cached_usd":0.0,"output_usd":0.02997,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":16.280218707979657}
{"item_id":"semeval2013.d008.s009.t002","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":2,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":3,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":4,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":5,"sense_key":"country%1:15:00::","synset_id":"country.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"country%1:15:00::","call_ids":["semeval2013.d008.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"country%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":2600,"reasoning_output_tokens":2561},"cost":{"currency":"USD","total_usd":0.08076,"input_uncached_usd":0.0027600000000000003,"input_cached_usd":0.0,"output_usd":0.078,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":61.56111504195724}
{"item_id":"semeval2013.d008.s009.t003","gold_sense_keys":["south_america%1:17:00::"],"candidates":[{"index":1,"sense_key":"south_america%1:14:00::","synset_id":"south_america.n.02"},{"index":2,"sense_key":"south_america%1:17:00::","synset_id":"south_america.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"south_america%1:17:00::","call_ids":["semeval2013.d008.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"south_america%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":124},"cost":{"currency":"USD","total_usd":0.006085,"input_uncached_usd":0.0019150000000000003,"input_cached_usd":0.0,"output_usd":0.00417,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8621430000057444}
{"item_id":"semeval2013.d008.s009.t004","gold_sense_keys":["table%1:14:00::"],"candidates":[{"index":1,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":2,"sense_key":"table%1:14:00::","synset_id":"table.n.01"},{"index":3,"sense_key":"table%1:06:02::","synset_id":"table.n.03"},{"index":4,"sense_key":"table%1:13:01::","synset_id":"board.n.04"},{"index":5,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":6,"sense_key":"table%1:14:01::","synset_id":"table.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"table%1:14:00::","call_ids":["semeval2013.d008.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"table%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.00474,"input_uncached_usd":0.0027600000000000003,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.702963792020455}
{"item_id":"semeval2013.d008.s010.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s010.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.005955,"input_uncached_usd":0.002475,"input_cached_usd":0.0,"output_usd":0.00348,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.520362125011161}
{"item_id":"semeval2013.d008.s010.t004","gold_sense_keys":["support%1:04:00::","support%1:09:00::"],"candidates":[{"index":1,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":2,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":3,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":4,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":7,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":8,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":9,"sense_key":"support%1:21:01::","synset_id":"support.n.11"},{"index":10,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":11,"sense_key":"support%1:04:03::","synset_id":"support.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"support%1:09:00::","call_ids":["semeval2013.d008.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"support%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":812,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0199,"input_uncached_usd":0.00406,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.810401291935705}
{"item_id":"semeval2013.d008.s010.t006","gold_sense_keys":["low%1:07:00::"],"candidates":[{"index":1,"sense_key":"low%1:07:00::","synset_id":"low.n.03"},{"index":2,"sense_key":"low%1:26:00::","synset_id":"low.n.01"},{"index":3,"sense_key":"low%1:06:00::","synset_id":"first_gear.n.01"},{"index":4,"sense_key":"low%1:18:00::","synset_id":"low.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"low%1:07:00::","call_ids":["semeval2013.d008.s010.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"low%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004385,"input_uncached_usd":0.002405,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8228375830221921}
{"item_id":"semeval2013.d008.s010.t007","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:14:00::","synset_id":"class.n.06"},{"index":2,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":3,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":4,"sense_key":"year%1:28:02::","synset_id":"year.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s010.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.00553,"input_uncached_usd":0.0024700000000000004,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.45860129094217}
{"item_id":"semeval2013.d008.s011.t001","gold_sense_keys":["economy%1:14:00::"],"candidates":[{"index":1,"sense_key":"economy%1:14:00::","synset_id":"economy.n.01"},{"index":2,"sense_key":"economy%1:04:00::","synset_id":"economy.n.04"},{"index":3,"sense_key":"economy%1:09:01::","synset_id":"economy.n.02"},{"index":4,"sense_key":"economy%1:07:00::","synset_id":"economy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"economy%1:14:00::","call_ids":["semeval2013.d008.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"economy%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.004595,"input_uncached_usd":0.0023750000000000004,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.707899833098054}
{"item_id":"semeval2013.d008.s011.t002","gold_sense_keys":["drop%1:07:00::"],"candidates":[{"index":1,"sense_key":"drop%1:06:01::","synset_id":"drop_curtain.n.01"},{"index":2,"sense_key":"drop%1:25:00::","synset_id":"drop.n.01"},{"index":3,"sense_key":"drop%1:04:00::","synset_id":"drop.n.09"},{"index":4,"sense_key":"drop%1:06:00::","synset_id":"drop.n.08"},{"index":5,"sense_key":"drop%1:11:00::","synset_id":"drop.n.06"},{"index":6,"sense_key":"drop%1:23:00::","synset_id":"drop.n.02"},{"index":7,"sense_key":"drop%1:15:00::","synset_id":"drop.n.05"},{"index":8,"sense_key":"drop%1:07:00::","synset_id":"drop.n.03"},{"index":9,"sense_key":"drop%1:17:00::","synset_id":"cliff.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"drop%1:07:00::","call_ids":["semeval2013.d008.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"drop%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":712,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.0053300000000000005,"input_uncached_usd":0.0035600000000000002,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":58.772389417048544}
{"item_id":"semeval2013.d008.s011.t003","gold_sense_keys":["output%1:04:00::","output%1:06:01::","output%1:23:00::"],"candidates":[{"index":1,"sense_key":"output%1:23:00::","synset_id":"output.n.04"},{"index":2,"sense_key":"output%1:06:00::","synset_id":"end_product.n.01"},{"index":3,"sense_key":"output%1:04:00::","synset_id":"output.n.02"},{"index":4,"sense_key":"output%1:06:01::","synset_id":"output.n.05"},{"index":5,"sense_key":"output%1:10:02::","synset_id":"output_signal.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"output%1:23:00::","call_ids":["semeval2013.d008.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"output%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":320,"reasoning_output_tokens":304},"cost":{"currency":"USD","total_usd":0.01202,"input_uncached_usd":0.0024200000000000003,"input_cached_usd":0.0,"output_usd":0.009600000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.598355541937053}
{"item_id":"semeval2013.d008.s011.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":2,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.00456,"input_uncached_usd":0.00255,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8670974581036717}
{"item_id":"semeval2013.d008.s012.t000","gold_sense_keys":["half%1:23:00::"],"candidates":[{"index":1,"sense_key":"half%1:23:00::","synset_id":"one-half.n.01"},{"index":2,"sense_key":"half%1:28:00::","synset_id":"half.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"half%1:23:00::","call_ids":["semeval2013.d008.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"half%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.003685,"input_uncached_usd":0.002095,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.059493500040844}
{"item_id":"semeval2013.d008.s012.t001","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"},{"index":2,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"respondent%1:18:00::","call_ids":["semeval2013.d008.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"respondent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.0035450000000000004,"input_uncached_usd":0.0020450000000000004,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4473907500505447}
{"item_id":"semeval2013.d008.s012.t003","gold_sense_keys":["brazil%1:15:00::"],"candidates":[{"index":1,"sense_key":"brazil%1:15:00::","synset_id":"brazil.n.01"},{"index":2,"sense_key":"brazil%1:13:00::","synset_id":"brazil_nut.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brazil%1:15:00::","call_ids":["semeval2013.d008.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brazil%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":431,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.0036850000000000003,"input_uncached_usd":0.0021550000000000002,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8843939590733498}
{"item_id":"semeval2013.d008.s012.t005","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":4,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":2592,"reasoning_output_tokens":2552},"cost":{"currency":"USD","total_usd":0.08070000000000001,"input_uncached_usd":0.0029400000000000003,"input_cached_usd":0.0,"output_usd":0.07776,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":43.47346004191786}
{"item_id":"semeval2013.d008.s012.t006","gold_sense_keys":["democrat%1:18:00::"],"candidates":[{"index":1,"sense_key":"democrat%1:18:01::","synset_id":"democrat.n.01"},{"index":2,"sense_key":"democrat%1:18:00::","synset_id":"democrat.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democrat%1:18:00::","call_ids":["semeval2013.d008.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democrat%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":392,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.0037600000000000003,"input_uncached_usd":0.0019600000000000004,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.989684165921062}
{"item_id":"semeval2013.d008.s012.t007","gold_sense_keys":["average%1:09:00::"],"candidates":[{"index":1,"sense_key":"average%1:24:00::","synset_id":"average.n.02"},{"index":2,"sense_key":"average%1:09:01::","synset_id":"average.n.03"},{"index":3,"sense_key":"average%1:09:00::","synset_id":"average.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"average%1:09:00::","call_ids":["semeval2013.d008.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"average%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":86},"cost":{"currency":"USD","total_usd":0.005295,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0588725419947878}
{"item_id":"semeval2013.d008.s012.t008","gold_sense_keys":["chart%1:10:00::"],"candidates":[{"index":1,"sense_key":"chart%1:10:00::","synset_id":"chart.n.01"},{"index":2,"sense_key":"chart%1:06:00::","synset_id":"chart.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"chart%1:10:00::","call_ids":["semeval2013.d008.s012.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"chart%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.003945000000000001,"input_uncached_usd":0.001935,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7692673339042813}
{"item_id":"semeval2013.d008.s013.t000","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"},{"index":2,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"respondent%1:18:00::","call_ids":["semeval2013.d008.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"respondent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":40,"reasoning_output_tokens":25},"cost":{"currency":"USD","total_usd":0.0034200000000000003,"input_uncached_usd":0.00222,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3194802920334041}
{"item_id":"semeval2013.d008.s013.t002","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":4,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":5,"sense_key":"country%1:14:00::","synset_id":"state.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.009365000000000002,"input_uncached_usd":0.003125,"input_cached_usd":0.0,"output_usd":0.00624,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.228698624996468}
{"item_id":"semeval2013.d008.s013.t003","gold_sense_keys":["democracy%1:14:00::"],"candidates":[{"index":1,"sense_key":"democracy%1:09:01::","synset_id":"majority_rule.n.01"},{"index":2,"sense_key":"democracy%1:14:00::","synset_id":"democracy.n.02"},{"index":3,"sense_key":"democracy%1:09:00::","synset_id":"democracy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democracy%1:14:00::","call_ids":["semeval2013.d008.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democracy%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":106},"cost":{"currency":"USD","total_usd":0.006115000000000001,"input_uncached_usd":0.0024850000000000002,"input_cached_usd":0.0,"output_usd":0.00363,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6294666660251096}
{"item_id":"semeval2013.d008.s013.t005","gold_sense_keys":["figure%1:07:01::"],"candidates":[{"index":1,"sense_key":"figure%1:23:00::","synset_id":"digit.n.01"},{"index":2,"sense_key":"figure%1:25:00::","synset_id":"figure.n.06"},{"index":3,"sense_key":"figure%1:09:00::","synset_id":"figure.n.11"},{"index":4,"sense_key":"figure%1:08:00::","synset_id":"human_body.n.01"},{"index":5,"sense_key":"figure%1:04:00::","synset_id":"figure.n.13"},{"index":6,"sense_key":"figure%1:07:00::","synset_id":"figure.n.08"},{"index":7,"sense_key":"figure%1:06:01::","synset_id":"design.n.04"},{"index":8,"sense_key":"figure%1:18:00::","synset_id":"name.n.04"},{"index":9,"sense_key":"figure%1:06:00::","synset_id":"figure.n.04"},{"index":10,"sense_key":"figure%1:10:00::","synset_id":"figure.n.01"},{"index":11,"sense_key":"figure%1:07:01::","synset_id":"number.n.01"},{"index":12,"sense_key":"figure%1:10:01::","synset_id":"trope.n.01"},{"index":13,"sense_key":"figure%1:21:00::","synset_id":"figure.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"figure%1:07:01::","call_ids":["semeval2013.d008.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"figure%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":905,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":79},"cost":{"currency":"USD","total_usd":0.0073750000000000005,"input_uncached_usd":0.004525,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9028462499845773}
{"item_id":"semeval2013.d008.s013.t006","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s013.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":571,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.004955,"input_uncached_usd":0.0028550000000000003,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8712720000185072}
{"item_id":"semeval2013.d008.s013.t007","gold_sense_keys":["increase%1:07:00::","increase%1:11:00::"],"candidates":[{"index":1,"sense_key":"increase%1:04:00::","synset_id":"increase.n.05"},{"index":2,"sense_key":"increase%1:11:00::","synset_id":"increase.n.02"},{"index":3,"sense_key":"increase%1:07:00::","synset_id":"increase.n.04"},{"index":4,"sense_key":"increase%1:22:00::","synset_id":"increase.n.03"},{"index":5,"sense_key":"increase%1:23:00::","synset_id":"addition.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"increase%1:07:00::","call_ids":["semeval2013.d008.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"increase%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":439,"reasoning_output_tokens":423},"cost":{"currency":"USD","total_usd":0.016094999999999998,"input_uncached_usd":0.002925,"input_cached_usd":0.0,"output_usd":0.013170000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.024593750014901}
{"item_id":"semeval2013.d008.s013.t008","gold_sense_keys":["chart%1:10:00::"],"candidates":[{"index":1,"sense_key":"chart%1:10:00::","synset_id":"chart.n.01"},{"index":2,"sense_key":"chart%1:06:00::","synset_id":"chart.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"chart%1:10:00::","call_ids":["semeval2013.d008.s013.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"chart%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":421,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.003905,"input_uncached_usd":0.002105,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.541060083080083}
{"item_id":"semeval2013.d008.s014.t000","gold_sense_keys":["confidence%1:12:00::","confidence%1:26:02::"],"candidates":[{"index":1,"sense_key":"confidence%1:12:00::","synset_id":"confidence.n.02"},{"index":2,"sense_key":"confidence%1:26:03::","synset_id":"confidence.n.04"},{"index":3,"sense_key":"confidence%1:26:02::","synset_id":"confidence.n.03"},{"index":4,"sense_key":"confidence%1:10:00::","synset_id":"confidence.n.05"},{"index":5,"sense_key":"confidence%1:09:00::","synset_id":"assurance.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"confidence%1:12:00::","call_ids":["semeval2013.d008.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"confidence%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":199,"reasoning_output_tokens":183},"cost":{"currency":"USD","total_usd":0.008975,"input_uncached_usd":0.0030050000000000003,"input_cached_usd":0.0,"output_usd":0.0059700000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.162578666000627}
{"item_id":"semeval2013.d008.s014.t001","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":2,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":5,"sense_key":"country%1:14:00::","synset_id":"state.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":181,"reasoning_output_tokens":165},"cost":{"currency":"USD","total_usd":0.008550000000000002,"input_uncached_usd":0.0031200000000000004,"input_cached_usd":0.0,"output_usd":0.00543,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5454112919978797}
{"item_id":"semeval2013.d008.s014.t002","gold_sense_keys":["congress%1:14:02::"],"candidates":[{"index":1,"sense_key":"congress%1:14:00::","synset_id":"congress.n.02"},{"index":2,"sense_key":"congress%1:04:01::","synset_id":"sexual_intercourse.n.01"},{"index":3,"sense_key":"congress%1:14:01::","synset_id":"congress.n.01"},{"index":4,"sense_key":"congress%1:14:02::","synset_id":"congress.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"congress%1:14:02::","call_ids":["semeval2013.d008.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"congress%1:14:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.005545,"input_uncached_usd":0.002605,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0015214999439195}
{"item_id":"semeval2013.d008.s014.t003","gold_sense_keys":["government%1:14:00::"],"candidates":[{"index":1,"sense_key":"government%1:09:00::","synset_id":"government.n.03"},{"index":2,"sense_key":"government%1:14:00::","synset_id":"government.n.01"},{"index":3,"sense_key":"government%1:09:01::","synset_id":"politics.n.02"},{"index":4,"sense_key":"government%1:04:00::","synset_id":"government.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"government%1:14:00::","call_ids":["semeval2013.d008.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"government%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":554,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":118},"cost":{"currency":"USD","total_usd":0.006790000000000001,"input_uncached_usd":0.0027700000000000003,"input_cached_usd":0.0,"output_usd":0.00402,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.588355291984044}
{"item_id":"semeval2013.d008.s014.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":2,"sense_key":"year%1:14:00::","synset_id":"class.n.06"},{"index":3,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":4,"sense_key":"year%1:28:02::","synset_id":"year.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.0054,"input_uncached_usd":0.00285,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6201194169698283}
{"item_id":"semeval2013.d008.s014.t005","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":2,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":3,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":4,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":5,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":6,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":7,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":8,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":9,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":10,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":11,"sense_key":"case%1:06:01::","synset_id":"case.n.20"},{"index":12,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":13,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":14,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":15,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":16,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":17,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":18,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":19,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":20,"sense_key":"case%1:09:00::","synset_id":"case.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"case%1:11:00::","call_ids":["semeval2013.d008.s014.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"case%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1236,"cached_input_tokens":0,"output_tokens":240,"reasoning_output_tokens":224},"cost":{"currency":"USD","total_usd":0.013380000000000001,"input_uncached_usd":0.006180000000000001,"input_cached_usd":0.0,"output_usd":0.0072,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4357722910353914}
{"item_id":"semeval2013.d008.s015.t000","gold_sense_keys":["decade%1:28:00::"],"candidates":[{"index":1,"sense_key":"decade%1:23:00::","synset_id":"ten.n.01"},{"index":2,"sense_key":"decade%1:28:00::","synset_id":"decade.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"decade%1:28:00::","call_ids":["semeval2013.d008.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"decade%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00408,"input_uncached_usd":0.00231,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.526500582927838}
{"item_id":"semeval2013.d008.s015.t001","gold_sense_keys":["woe%1:26:00::"],"candidates":[{"index":1,"sense_key":"woe%1:12:00::","synset_id":"woe.n.02"},{"index":2,"sense_key":"woe%1:26:00::","synset_id":"suffering.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"woe%1:26:00::","call_ids":["semeval2013.d008.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"woe%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.004655,"input_uncached_usd":0.0021950000000000003,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3246336670126766}
{"item_id":"semeval2013.d008.s015.t003","gold_sense_keys":["support%1:04:00::","support%1:09:00::"],"candidates":[{"index":1,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":2,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":3,"sense_key":"support%1:21:01::","synset_id":"support.n.11"},{"index":4,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":5,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":6,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":7,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":8,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":9,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":10,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":11,"sense_key":"support%1:04:04::","synset_id":"support.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"support%1:09:00::","call_ids":["semeval2013.d008.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"support%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":894,"cached_input_tokens":0,"output_tokens":519,"reasoning_output_tokens":503},"cost":{"currency":"USD","total_usd":0.020040000000000002,"input_uncached_usd":0.00447,"input_cached_usd":0.0,"output_usd":0.01557,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.249077416025102}
{"item_id":"semeval2013.d008.s016.t001","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:14:00::","synset_id":"state.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01885,"input_uncached_usd":0.00301,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.060483791050501}
{"item_id":"semeval2013.d008.s016.t002","gold_sense_keys":["recession%1:26:00::"],"candidates":[{"index":1,"sense_key":"recession%1:26:00::","synset_id":"recession.n.01"},{"index":2,"sense_key":"recession%1:25:00::","synset_id":"recess.n.02"},{"index":3,"sense_key":"recession%1:14:00::","synset_id":"recession.n.03"},{"index":4,"sense_key":"recession%1:04:00::","synset_id":"receding.n.02"},{"index":5,"sense_key":"recession%1:04:01::","synset_id":"recession.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recession%1:26:00::","call_ids":["semeval2013.d008.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recession%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":40,"reasoning_output_tokens":25},"cost":{"currency":"USD","total_usd":0.0038200000000000005,"input_uncached_usd":0.0026200000000000004,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.446192792034708}
{"item_id":"semeval2013.d008.s016.t003","gold_sense_keys":["recovery%1:11:00::"],"candidates":[{"index":1,"sense_key":"recovery%1:11:00::","synset_id":"recovery.n.01"},{"index":2,"sense_key":"recovery%1:04:00::","synset_id":"recovery.n.03"},{"index":3,"sense_key":"recovery%1:22:00::","synset_id":"convalescence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recovery%1:11:00::","call_ids":["semeval2013.d008.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recovery%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.004875000000000001,"input_uncached_usd":0.0023250000000000002,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.028115624911152}
{"item_id":"semeval2013.d008.s017.t001","gold_sense_keys":["safety_net%1:10:00::"],"candidates":[{"index":1,"sense_key":"safety_net%1:06:00::","synset_id":"safety_net.n.02"},{"index":2,"sense_key":"safety_net%1:10:00::","synset_id":"safety_net.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"safety_net%1:10:00::","call_ids":["semeval2013.d008.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"safety_net%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":438,"cached_input_tokens":0,"output_tokens":34,"reasoning_output_tokens":19},"cost":{"currency":"USD","total_usd":0.00321,"input_uncached_usd":0.00219,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2455437500029802}
{"item_id":"semeval2013.d008.s018.t002","gold_sense_keys":["week%1:28:02::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:02::","synset_id":"week.n.03"},{"index":3,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"week%1:28:02::","call_ids":["semeval2013.d008.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"week%1:28:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":190,"reasoning_output_tokens":174},"cost":{"currency":"USD","total_usd":0.00802,"input_uncached_usd":0.00232,"input_cached_usd":0.0,"output_usd":0.0057,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.898468125029467}
{"item_id":"semeval2013.d008.s018.t003","gold_sense_keys":["proportion%1:24:00::"],"candidates":[{"index":1,"sense_key":"proportion%1:24:01::","synset_id":"proportion.n.04"},{"index":2,"sense_key":"proportion%1:07:01::","synset_id":"proportion.n.05"},{"index":3,"sense_key":"proportion%1:24:00::","synset_id":"proportion.n.01"},{"index":4,"sense_key":"proportion%1:25:00::","synset_id":"symmetry.n.02"},{"index":5,"sense_key":"proportion%1:07:00::","synset_id":"proportion.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"proportion%1:24:00::","call_ids":["semeval2013.d008.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"proportion%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.006110000000000001,"input_uncached_usd":0.0029600000000000004,"input_cached_usd":0.0,"output_usd":0.00315,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8078665839275345}
{"item_id":"semeval2013.d008.s018.t005","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:14:00::","synset_id":"class.n.06"},{"index":2,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":3,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":4,"sense_key":"year%1:28:00::","synset_id":"year.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":550,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.00584,"input_uncached_usd":0.0027500000000000003,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.594664124888368}
{"item_id":"semeval2013.d008.s019.t000","gold_sense_keys":["mood%1:12:00::","mood%1:26:00::"],"candidates":[{"index":1,"sense_key":"mood%1:24:00::","synset_id":"mood.n.03"},{"index":2,"sense_key":"mood%1:26:00::","synset_id":"climate.n.02"},{"index":3,"sense_key":"mood%1:12:00::","synset_id":"temper.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mood%1:26:00::","call_ids":["semeval2013.d008.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mood%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.00478,"input_uncached_usd":0.0024100000000000002,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2305560830282047}
{"item_id":"semeval2013.d008.s019.t001","gold_sense_keys":["country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":2,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":3,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":4,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":5,"sense_key":"country%1:14:00::","synset_id":"state.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"country%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":2215,"reasoning_output_tokens":2174},"cost":{"currency":"USD","total_usd":0.069395,"input_uncached_usd":0.002945,"input_cached_usd":0.0,"output_usd":0.06645,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":43.026892583002336}
{"item_id":"semeval2013.d008.s019.t002","gold_sense_keys":["country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":2,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":3,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:14:00::","synset_id":"state.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"country%1:14:01::","call_ids":["semeval2013.d008.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"country%1:14:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":8140,"reasoning_output_tokens":8034},"cost":{"currency":"USD","total_usd":0.247145,"input_uncached_usd":0.002945,"input_cached_usd":0.0,"output_usd":0.2442,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":149.15462325001135}
{"item_id":"semeval2013.d008.s019.t003","gold_sense_keys":["chart%1:10:00::"],"candidates":[{"index":1,"sense_key":"chart%1:06:00::","synset_id":"chart.n.02"},{"index":2,"sense_key":"chart%1:10:00::","synset_id":"chart.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"chart%1:10:00::","call_ids":["semeval2013.d008.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"chart%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.003585,"input_uncached_usd":0.001935,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":65.18423675000668}
{"item_id":"semeval2013.d008.s020.t000","gold_sense_keys":["brazil%1:15:00::"],"candidates":[{"index":1,"sense_key":"brazil%1:15:00::","synset_id":"brazil.n.01"},{"index":2,"sense_key":"brazil%1:13:00::","synset_id":"brazil_nut.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brazil%1:15:00::","call_ids":["semeval2013.d008.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brazil%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":411,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.003555,"input_uncached_usd":0.002055,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.387616042047739}
{"item_id":"semeval2013.d008.s020.t003","gold_sense_keys":["president%1:18:00::"],"candidates":[{"index":1,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":4,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":5,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"president%1:18:00::","call_ids":["semeval2013.d008.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"president%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":558,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00474,"input_uncached_usd":0.0027900000000000004,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8895905419485644}
{"item_id":"semeval2013.d008.s020.t004","gold_sense_keys":["sense%1:09:05::"],"candidates":[{"index":1,"sense_key":"sense%1:09:04::","synset_id":"common_sense.n.01"},{"index":2,"sense_key":"sense%1:10:00::","synset_id":"sense.n.02"},{"index":3,"sense_key":"sense%1:09:05::","synset_id":"sense.n.01"},{"index":4,"sense_key":"sense%1:09:06::","synset_id":"sense.n.05"},{"index":5,"sense_key":"sense%1:09:02::","synset_id":"sense.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"sense%1:09:05::","call_ids":["semeval2013.d008.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"sense%1:09:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.004195000000000001,"input_uncached_usd":0.0028150000000000002,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.654566292068921}
{"item_id":"semeval2013.d008.s020.t005","gold_sense_keys":["progress%1:04:01::"],"candidates":[{"index":1,"sense_key":"progress%1:04:00::","synset_id":"progress.n.02"},{"index":2,"sense_key":"progress%1:11:00::","synset_id":"progress.n.03"},{"index":3,"sense_key":"progress%1:04:01::","synset_id":"advancement.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"progress%1:04:01::","call_ids":["semeval2013.d008.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"progress%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.00455,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.949590416974388}
{"item_id":"semeval2013.d008.s021.t002","gold_sense_keys":["recession%1:26:00::"],"candidates":[{"index":1,"sense_key":"recession%1:04:00::","synset_id":"receding.n.02"},{"index":2,"sense_key":"recession%1:26:00::","synset_id":"recession.n.01"},{"index":3,"sense_key":"recession%1:25:00::","synset_id":"recess.n.02"},{"index":4,"sense_key":"recession%1:04:01::","synset_id":"recession.n.04"},{"index":5,"sense_key":"recession%1:14:00::","synset_id":"recession.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"recession%1:26:00::","call_ids":["semeval2013.d008.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"recession%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":33,"reasoning_output_tokens":17},"cost":{"currency":"USD","total_usd":0.003565,"input_uncached_usd":0.002575,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8672560000559315}
{"item_id":"semeval2013.d008.s021.t003","gold_sense_keys":["crime%1:04:00::"],"candidates":[{"index":1,"sense_key":"crime%1:04:00::","synset_id":"crime.n.01"},{"index":2,"sense_key":"crime%1:04:01::","synset_id":"crime.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crime%1:04:00::","call_ids":["semeval2013.d008.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crime%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":80},"cost":{"currency":"USD","total_usd":0.005015,"input_uncached_usd":0.0021650000000000003,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.086541000055149}
{"item_id":"semeval2013.d008.s022.t000","gold_sense_keys":["time%1:11:00::"],"candidates":[{"index":1,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":2,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":3,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":4,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":7,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"},{"index":8,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":9,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":10,"sense_key":"time%1:11:00::","synset_id":"time.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"time%1:11:00::","call_ids":["semeval2013.d008.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"time%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.0059700000000000005,"input_uncached_usd":0.0038100000000000005,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6479429169557989}
{"item_id":"semeval2013.d008.s022.t001","gold_sense_keys":["poll%1:09:00::"],"candidates":[{"index":1,"sense_key":"poll%1:09:00::","synset_id":"poll.n.01"},{"index":2,"sense_key":"poll%1:05:00::","synset_id":"poll.n.04"},{"index":3,"sense_key":"poll%1:08:01::","synset_id":"pate.n.02"},{"index":4,"sense_key":"poll%1:08:02::","synset_id":"poll.n.03"},{"index":5,"sense_key":"poll%1:04:00::","synset_id":"poll.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"poll%1:09:00::","call_ids":["semeval2013.d008.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"poll%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.00487,"input_uncached_usd":0.00253,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9555941669968888}
{"item_id":"semeval2013.d008.s022.t002","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"},{"index":2,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"respondent%1:18:00::","call_ids":["semeval2013.d008.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"respondent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.0037800000000000004,"input_uncached_usd":0.0021300000000000004,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.884069832973182}
{"item_id":"semeval2013.d008.s022.t004","gold_sense_keys":["whole%1:09:00::"],"candidates":[{"index":1,"sense_key":"whole%1:03:00::","synset_id":"whole.n.02"},{"index":2,"sense_key":"whole%1:09:00::","synset_id":"whole.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"whole%1:09:00::","call_ids":["semeval2013.d008.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"whole%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":81},"cost":{"currency":"USD","total_usd":0.00516,"input_uncached_usd":0.0022500000000000003,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.182662041974254}
{"item_id":"semeval2013.d008.s022.t005","gold_sense_keys":["crime%1:04:00::"],"candidates":[{"index":1,"sense_key":"crime%1:04:01::","synset_id":"crime.n.02"},{"index":2,"sense_key":"crime%1:04:00::","synset_id":"crime.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"crime%1:04:00::","call_ids":["semeval2013.d008.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"crime%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.004025,"input_uncached_usd":0.002225,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4619755420135334}
{"item_id":"semeval2013.d008.s022.t007","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"problem%1:26:00::","call_ids":["semeval2013.d008.s022.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"problem%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01823,"input_uncached_usd":0.00239,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.97302941698581}
{"item_id":"semeval2013.d008.s022.t008","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":2,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":3,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":4,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":5,"sense_key":"country%1:15:00::","synset_id":"country.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s022.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":388,"reasoning_output_tokens":373},"cost":{"currency":"USD","total_usd":0.01467,"input_uncached_usd":0.00303,"input_cached_usd":0.0,"output_usd":0.011640000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.684728291002102}
{"item_id":"semeval2013.d008.s022.t009","gold_sense_keys":["chart%1:10:00::"],"candidates":[{"index":1,"sense_key":"chart%1:10:00::","synset_id":"chart.n.01"},{"index":2,"sense_key":"chart%1:06:00::","synset_id":"chart.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"chart%1:10:00::","call_ids":["semeval2013.d008.s022.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"chart%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00391,"input_uncached_usd":0.00202,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8213876659283414}
{"item_id":"semeval2013.d008.s023.t000","gold_sense_keys":["relative%1:18:00::"],"candidates":[{"index":1,"sense_key":"relative%1:18:00::","synset_id":"relative.n.01"},{"index":2,"sense_key":"relative%1:05:00::","synset_id":"relative.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"relative%1:18:00::","call_ids":["semeval2013.d008.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"relative%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.004020000000000001,"input_uncached_usd":0.0024600000000000004,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5918841250240803}
{"item_id":"semeval2013.d008.s023.t001","gold_sense_keys":["victim%1:18:00::"],"candidates":[{"index":1,"sense_key":"victim%1:18:01::","synset_id":"victim.n.02"},{"index":2,"sense_key":"victim%1:18:00::","synset_id":"victim.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"victim%1:18:00::","call_ids":["semeval2013.d008.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"victim%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.00504,"input_uncached_usd":0.0022500000000000003,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.364710291964002}
{"item_id":"semeval2013.d008.s023.t002","gold_sense_keys":["crime%1:04:00::"],"candidates":[{"index":1,"sense_key":"crime%1:04:00::","synset_id":"crime.n.01"},{"index":2,"sense_key":"crime%1:04:01::","synset_id":"crime.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crime%1:04:00::","call_ids":["semeval2013.d008.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crime%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.005135,"input_uncached_usd":0.002405,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3454831250710413}
{"item_id":"semeval2013.d008.s023.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:14:00::","synset_id":"class.n.06"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:28:01::","synset_id":"year.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.004555,"input_uncached_usd":0.002935,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.967026999918744}
{"item_id":"semeval2013.d008.s023.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:14:00::","synset_id":"class.n.06"},{"index":4,"sense_key":"year%1:28:00::","synset_id":"year.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.00555,"input_uncached_usd":0.0029400000000000003,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0766491659451276}
{"item_id":"semeval2013.d008.s023.t005","gold_sense_keys":["figure%1:07:01::"],"candidates":[{"index":1,"sense_key":"figure%1:10:00::","synset_id":"figure.n.01"},{"index":2,"sense_key":"figure%1:09:00::","synset_id":"figure.n.11"},{"index":3,"sense_key":"figure%1:08:00::","synset_id":"human_body.n.01"},{"index":4,"sense_key":"figure%1:07:00::","synset_id":"figure.n.08"},{"index":5,"sense_key":"figure%1:07:01::","synset_id":"number.n.01"},{"index":6,"sense_key":"figure%1:10:01::","synset_id":"trope.n.01"},{"index":7,"sense_key":"figure%1:23:00::","synset_id":"digit.n.01"},{"index":8,"sense_key":"figure%1:06:01::","synset_id":"design.n.04"},{"index":9,"sense_key":"figure%1:06:00::","synset_id":"figure.n.04"},{"index":10,"sense_key":"figure%1:25:00::","synset_id":"figure.n.06"},{"index":11,"sense_key":"figure%1:18:00::","synset_id":"name.n.04"},{"index":12,"sense_key":"figure%1:04:00::","synset_id":"figure.n.13"},{"index":13,"sense_key":"figure%1:21:00::","synset_id":"figure.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"figure%1:07:01::","call_ids":["semeval2013.d008.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"figure%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":922,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.00809,"input_uncached_usd":0.00461,"input_cached_usd":0.0,"output_usd":0.00348,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8989208330167457}
{"item_id":"semeval2013.d008.s024.t001","gold_sense_keys":["support%1:04:00::","support%1:09:00::"],"candidates":[{"index":1,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":2,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":3,"sense_key":"support%1:21:01::","synset_id":"support.n.11"},{"index":4,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":5,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":6,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":7,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":8,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":9,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":10,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":11,"sense_key":"support%1:04:03::","synset_id":"support.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"support%1:09:00::","call_ids":["semeval2013.d008.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"support%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":889,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020285,"input_uncached_usd":0.004445,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.40561829204671}
{"item_id":"semeval2013.d008.s024.t004","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"},{"index":2,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"respondent%1:18:00::","call_ids":["semeval2013.d008.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"respondent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.003815,"input_uncached_usd":0.002225,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.965082582901232}
{"item_id":"semeval2013.d008.s024.t007","gold_sense_keys":["development%1:04:01::","development%1:22:02::","development%1:26:00::"],"candidates":[{"index":1,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":2,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":3,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":4,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":5,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":6,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":7,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":8,"sense_key":"development%1:10:00::","synset_id":"development.n.09"},{"index":9,"sense_key":"development%1:22:00::","synset_id":"development.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"development%1:22:02::","call_ids":["semeval2013.d008.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"development%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":280,"reasoning_output_tokens":264},"cost":{"currency":"USD","total_usd":0.012660000000000001,"input_uncached_usd":0.004260000000000001,"input_cached_usd":0.0,"output_usd":0.0084,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.666045457939617}
{"item_id":"semeval2013.d008.s024.t008","gold_sense_keys":["chart%1:10:00::"],"candidates":[{"index":1,"sense_key":"chart%1:10:00::","synset_id":"chart.n.01"},{"index":2,"sense_key":"chart%1:06:00::","synset_id":"chart.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"chart%1:10:00::","call_ids":["semeval2013.d008.s024.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"chart%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":423,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004095,"input_uncached_usd":0.002115,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7147651250706986}
{"item_id":"semeval2013.d008.s025.t001","gold_sense_keys":["system%1:14:00::"],"candidates":[{"index":1,"sense_key":"system%1:08:00::","synset_id":"system.n.06"},{"index":2,"sense_key":"system%1:27:00::","synset_id":"system.n.03"},{"index":3,"sense_key":"system%1:07:00::","synset_id":"organization.n.05"},{"index":4,"sense_key":"system%1:14:00::","synset_id":"system.n.02"},{"index":5,"sense_key":"system%1:09:00::","synset_id":"system.n.04"},{"index":6,"sense_key":"system%1:09:02::","synset_id":"arrangement.n.03"},{"index":7,"sense_key":"system%1:06:00::","synset_id":"system.n.01"},{"index":8,"sense_key":"system%1:09:01::","synset_id":"system.n.07"},{"index":9,"sense_key":"system%1:08:01::","synset_id":"system.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"system%1:14:00::","call_ids":["semeval2013.d008.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"system%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":855,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.035655,"input_uncached_usd":0.004275,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":16.03033354098443}
{"item_id":"semeval2013.d008.s025.t002","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":2,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":3,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:00::","synset_id":"country.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":651,"cached_input_tokens":0,"output_tokens":305,"reasoning_output_tokens":289},"cost":{"currency":"USD","total_usd":0.012405,"input_uncached_usd":0.003255,"input_cached_usd":0.0,"output_usd":0.00915,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.937672917032614}
{"item_id":"semeval2013.d008.s025.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s025.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":90},"cost":{"currency":"USD","total_usd":0.006135,"input_uncached_usd":0.0029850000000000002,"input_cached_usd":0.0,"output_usd":0.00315,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.368496499955654}
{"item_id":"semeval2013.d008.s026.t000","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"},{"index":2,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"respondent%1:18:00::","call_ids":["semeval2013.d008.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"respondent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.0038250000000000003,"input_uncached_usd":0.002475,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2618665000190958}
{"item_id":"semeval2013.d008.s026.t002","gold_sense_keys":["view%1:09:02::","view%1:09:04::"],"candidates":[{"index":1,"sense_key":"view%1:09:02::","synset_id":"position.n.03"},{"index":2,"sense_key":"view%1:06:00::","synset_id":"scene.n.08"},{"index":3,"sense_key":"view%1:10:00::","synset_id":"opinion.n.02"},{"index":4,"sense_key":"view%1:09:04::","synset_id":"opinion.n.01"},{"index":5,"sense_key":"view%1:07:01::","synset_id":"view.n.10"},{"index":6,"sense_key":"view%1:04:00::","synset_id":"view.n.03"},{"index":7,"sense_key":"view%1:07:00::","synset_id":"horizon.n.02"},{"index":8,"sense_key":"view%1:09:00::","synset_id":"view.n.07"},{"index":9,"sense_key":"view%1:09:01::","synset_id":"view.n.02"},{"index":10,"sense_key":"view%1:15:00::","synset_id":"view.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"view%1:09:04::","call_ids":["semeval2013.d008.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"view%1:09:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":893,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020305,"input_uncached_usd":0.004465,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.357667083968408}
{"item_id":"semeval2013.d008.s026.t004","gold_sense_keys":["president%1:18:00::"],"candidates":[{"index":1,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:18:03::","synset_id":"president.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"president%1:18:00::","call_ids":["semeval2013.d008.s026.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"president%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.005215,"input_uncached_usd":0.0033250000000000003,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7152153749484569}
{"item_id":"semeval2013.d008.s026.t005","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":3,"sense_key":"company%1:14:05::","synset_id":"company.n.09"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":6,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":7,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":8,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":9,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d008.s026.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":777,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.005445,"input_uncached_usd":0.0038850000000000004,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3801985409809276}
{"item_id":"semeval2013.d008.s027.t000","gold_sense_keys":["brazil%1:15:00::"],"candidates":[{"index":1,"sense_key":"brazil%1:13:00::","synset_id":"brazil_nut.n.02"},{"index":2,"sense_key":"brazil%1:15:00::","synset_id":"brazil.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"brazil%1:15:00::","call_ids":["semeval2013.d008.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"brazil%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.00409,"input_uncached_usd":0.00265,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7317944589303806}
{"item_id":"semeval2013.d008.s027.t001","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":2,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":3,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":4,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":5,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":105},"cost":{"currency":"USD","total_usd":0.007065,"input_uncached_usd":0.003435,"input_cached_usd":0.0,"output_usd":0.00363,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.205217625014484}
{"item_id":"semeval2013.d008.s027.t002","gold_sense_keys":["leadership%1:04:00::","leadership%1:09:00::","leadership%1:26:00::"],"candidates":[{"index":1,"sense_key":"leadership%1:04:00::","synset_id":"leadership.n.01"},{"index":2,"sense_key":"leadership%1:26:00::","synset_id":"leadership.n.03"},{"index":3,"sense_key":"leadership%1:09:00::","synset_id":"leadership.n.04"},{"index":4,"sense_key":"leadership%1:14:00::","synset_id":"leadership.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"leadership%1:04:00::","call_ids":["semeval2013.d008.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"leadership%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":114},"cost":{"currency":"USD","total_usd":0.006785000000000001,"input_uncached_usd":0.0028850000000000004,"input_cached_usd":0.0,"output_usd":0.0039000000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6153097909409553}
{"item_id":"semeval2013.d008.s027.t004","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"},{"index":2,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"respondent%1:18:00::","call_ids":["semeval2013.d008.s027.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"respondent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":507,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.004215,"input_uncached_usd":0.0025350000000000004,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8007992920465767}
{"item_id":"semeval2013.d008.s027.t005","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s027.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.00536,"input_uncached_usd":0.00317,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6825610410887748}
{"item_id":"semeval2013.d008.s027.t006","gold_sense_keys":["united_states%1:15:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"united_states%1:15:00::","call_ids":["semeval2013.d008.s027.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"united_states%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":561,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004785,"input_uncached_usd":0.002805,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.834856832982041}
{"item_id":"semeval2013.d008.s027.t007","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":3,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s027.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":97},"cost":{"currency":"USD","total_usd":0.006525,"input_uncached_usd":0.0031650000000000003,"input_cached_usd":0.0,"output_usd":0.00336,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8356064580148086}
{"item_id":"semeval2013.d008.s027.t009","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:14:00::","synset_id":"class.n.06"},{"index":2,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":3,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":4,"sense_key":"year%1:28:00::","synset_id":"year.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s027.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.005775000000000001,"input_uncached_usd":0.0031650000000000003,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6180988339474425}
{"item_id":"semeval2013.d008.s028.t000","gold_sense_keys":["united_states%1:15:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"united_states%1:15:00::","call_ids":["semeval2013.d008.s028.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"united_states%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.00519,"input_uncached_usd":0.0026100000000000003,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8305600830353796}
{"item_id":"semeval2013.d008.s028.t001","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":5,"sense_key":"country%1:15:01::","synset_id":"country.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s028.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":116},"cost":{"currency":"USD","total_usd":0.0072,"input_uncached_usd":0.0032400000000000003,"input_cached_usd":0.0,"output_usd":0.00396,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2823465410619974}
{"item_id":"semeval2013.d008.s028.t002","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"},{"index":2,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"respondent%1:18:00::","call_ids":["semeval2013.d008.s028.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"respondent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":468,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00402,"input_uncached_usd":0.00234,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.615398250054568}
{"item_id":"semeval2013.d008.s028.t006","gold_sense_keys":["distinction%1:07:01::"],"candidates":[{"index":1,"sense_key":"distinction%1:26:00::","synset_id":"eminence.n.01"},{"index":2,"sense_key":"distinction%1:07:01::","synset_id":"distinction.n.03"},{"index":3,"sense_key":"distinction%1:07:00::","synset_id":"distinction.n.04"},{"index":4,"sense_key":"distinction%1:09:00::","synset_id":"differentiation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"distinction%1:07:01::","call_ids":["semeval2013.d008.s028.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"distinction%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":268,"reasoning_output_tokens":252},"cost":{"currency":"USD","total_usd":0.01082,"input_uncached_usd":0.0027800000000000004,"input_cached_usd":0.0,"output_usd":0.00804,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.974586499971338}
{"item_id":"semeval2013.d009.s000.t001","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":2,"sense_key":"law%1:14:01::","synset_id":"police.n.01"},{"index":3,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":4,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":5,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":6,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":7,"sense_key":"law%1:04:00::","synset_id":"law.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":73},"cost":{"currency":"USD","total_usd":0.005390000000000001,"input_uncached_usd":0.00272,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.848943666089326}
{"item_id":"semeval2013.d009.s000.t003","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":2,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":3,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":4,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":5,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":6,"sense_key":"court%1:04:00::","synset_id":"court.n.11"},{"index":7,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":8,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":11,"sense_key":"court%1:14:02::","synset_id":"court.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"court%1:14:00::","call_ids":["semeval2013.d009.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019135,"input_uncached_usd":0.003295,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.305913166957907}
{"item_id":"semeval2013.d009.s000.t004","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":2,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":3,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":4,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":5,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":6,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":7,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":8,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":9,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":10,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":11,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":12,"sense_key":"case%1:06:01::","synset_id":"case.n.20"},{"index":13,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":14,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":15,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":16,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":17,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":18,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":19,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":20,"sense_key":"case%1:18:04::","synset_id":"character.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1088,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":120},"cost":{"currency":"USD","total_usd":0.00952,"input_uncached_usd":0.00544,"input_cached_usd":0.0,"output_usd":0.00408,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.598273249925114}
{"item_id":"semeval2013.d009.s001.t000","gold_sense_keys":["day%1:28:03::"],"candidates":[{"index":1,"sense_key":"day%1:28:04::","synset_id":"day.n.05"},{"index":2,"sense_key":"day%1:28:05::","synset_id":"day.n.06"},{"index":3,"sense_key":"day%1:26:00::","synset_id":"day.n.09"},{"index":4,"sense_key":"day%1:28:01::","synset_id":"day.n.03"},{"index":5,"sense_key":"day%1:28:02::","synset_id":"day.n.04"},{"index":6,"sense_key":"day%1:28:07::","synset_id":"day.n.07"},{"index":7,"sense_key":"day%1:28:03::","synset_id":"day.n.02"},{"index":8,"sense_key":"day%1:28:00::","synset_id":"day.n.01"},{"index":9,"sense_key":"day%1:28:06::","synset_id":"sidereal_day.n.01"},{"index":10,"sense_key":"day%1:18:00::","synset_id":"day.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"day%1:28:03::","call_ids":["semeval2013.d009.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"day%1:28:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":713,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.034945,"input_uncached_usd":0.0035650000000000005,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":19.22664241702296}
{"item_id":"semeval2013.d009.s001.t001","gold_sense_keys":["supreme_court%1:14:00::"],"candidates":[{"index":1,"sense_key":"supreme_court%1:14:00::","synset_id":"supreme_court.n.01"},{"index":2,"sense_key":"supreme_court%1:14:01::","synset_id":"supreme_court.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"supreme_court%1:14:00::","call_ids":["semeval2013.d009.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"supreme_court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":350,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.0036699999999999997,"input_uncached_usd":0.00175,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8937478750012815}
{"item_id":"semeval2013.d009.s001.t002","gold_sense_keys":["question%1:10:01::"],"candidates":[{"index":1,"sense_key":"question%1:10:00::","synset_id":"question.n.01"},{"index":2,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"},{"index":3,"sense_key":"question%1:10:03::","synset_id":"question.n.03"},{"index":4,"sense_key":"question%1:10:04::","synset_id":"question.n.06"},{"index":5,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":6,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"question%1:10:01::","call_ids":["semeval2013.d009.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"question%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.004935,"input_uncached_usd":0.0027150000000000004,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8004999168915674}
{"item_id":"semeval2013.d009.s001.t005","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":2,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":3,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":4,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":5,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":6,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":7,"sense_key":"law%1:14:01::","synset_id":"police.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":566,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.00604,"input_uncached_usd":0.00283,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9849300000350922}
{"item_id":"semeval2013.d009.s001.t006","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"},{"index":2,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":3,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":4,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":401,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.0040750000000000005,"input_uncached_usd":0.0020050000000000003,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4498227910371497}
{"item_id":"semeval2013.d009.s001.t007","gold_sense_keys":["dinner%1:14:00::"],"candidates":[{"index":1,"sense_key":"dinner%1:13:00::","synset_id":"dinner.n.01"},{"index":2,"sense_key":"dinner%1:14:00::","synset_id":"dinner.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dinner%1:14:00::","call_ids":["semeval2013.d009.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dinner%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":344,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":141},"cost":{"currency":"USD","total_usd":0.006430000000000001,"input_uncached_usd":0.0017200000000000002,"input_cached_usd":0.0,"output_usd":0.00471,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.536819916917011}
{"item_id":"semeval2013.d009.s001.t009","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":4,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":5,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":6,"sense_key":"case%1:06:01::","synset_id":"case.n.20"},{"index":7,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":8,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":9,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":10,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":11,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":12,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":13,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":14,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":15,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":16,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":17,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":18,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":19,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":20,"sense_key":"case%1:10:02::","synset_id":"case.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1111,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.007415,"input_uncached_usd":0.005555,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5232435839716345}
{"item_id":"semeval2013.d009.s002.t002","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":3,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":4,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.003915,"input_uncached_usd":0.0022050000000000004,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1047007919987664}
{"item_id":"semeval2013.d009.s002.t003","gold_sense_keys":["conference%1:10:00::","conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"},{"index":2,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":3,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conference%1:14:00::","call_ids":["semeval2013.d009.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":138},"cost":{"currency":"USD","total_usd":0.006540000000000001,"input_uncached_usd":0.0019500000000000001,"input_cached_usd":0.0,"output_usd":0.00459,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3371500830398872}
{"item_id":"semeval2013.d009.s002.t004","gold_sense_keys":["petition%1:10:00::"],"candidates":[{"index":1,"sense_key":"petition%1:10:01::","synset_id":"prayer.n.02"},{"index":2,"sense_key":"petition%1:10:00::","synset_id":"request.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"petition%1:10:00::","call_ids":["semeval2013.d009.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"petition%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":353,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.0033250000000000003,"input_uncached_usd":0.001765,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0918942500138655}
{"item_id":"semeval2013.d009.s002.t005","gold_sense_keys":["review%1:04:02::","review%1:04:03::","review%1:09:00::"],"candidates":[{"index":1,"sense_key":"review%1:04:00::","synset_id":"follow-up.n.03"},{"index":2,"sense_key":"review%1:10:02::","synset_id":"revue.n.01"},{"index":3,"sense_key":"review%1:10:03::","synset_id":"review.n.06"},{"index":4,"sense_key":"review%1:21:00::","synset_id":"review.n.04"},{"index":5,"sense_key":"review%1:09:00::","synset_id":"reappraisal.n.01"},{"index":6,"sense_key":"review%1:04:02::","synset_id":"inspection.n.01"},{"index":7,"sense_key":"review%1:10:01::","synset_id":"review.n.02"},{"index":8,"sense_key":"review%1:10:00::","synset_id":"recapitulation.n.03"},{"index":9,"sense_key":"review%1:04:01::","synset_id":"review.n.09"},{"index":10,"sense_key":"review%1:04:03::","synset_id":"review.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"review%1:04:03::","call_ids":["semeval2013.d009.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"review%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.0062699999999999995,"input_uncached_usd":0.00333,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0831731249345466}
{"item_id":"semeval2013.d009.s003.t000","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":3,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":4,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":5,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":6,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":7,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":8,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":9,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":10,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":15,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":16,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":17,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":18,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":19,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1207,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.007925000000000001,"input_uncached_usd":0.006035,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4673620420508087}
{"item_id":"semeval2013.d009.s003.t002","gold_sense_keys":["suit%1:04:00::"],"candidates":[{"index":1,"sense_key":"suit%1:06:00::","synset_id":"suit.n.01"},{"index":2,"sense_key":"suit%1:10:01::","synset_id":"courtship.n.01"},{"index":3,"sense_key":"suit%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"suit%1:18:00::","synset_id":"suit.n.03"},{"index":5,"sense_key":"suit%1:10:00::","synset_id":"suit.n.05"},{"index":6,"sense_key":"suit%1:06:01::","synset_id":"suit.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"suit%1:04:00::","call_ids":["semeval2013.d009.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"suit%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":641,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.004915,"input_uncached_usd":0.0032050000000000004,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5073999169981107}
{"item_id":"semeval2013.d009.s003.t003","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":2,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":3,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":4,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":5,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"},{"index":6,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":7,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":8,"sense_key":"state%1:26:01::","synset_id":"state.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"state%1:14:01::","call_ids":["semeval2013.d009.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"state%1:14:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":4065,"reasoning_output_tokens":4006},"cost":{"currency":"USD","total_usd":0.125635,"input_uncached_usd":0.0036850000000000003,"input_cached_usd":0.0,"output_usd":0.12195,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":67.71938945900183}
{"item_id":"semeval2013.d009.s003.t005","gold_sense_keys":["overhaul%1:04:01::"],"candidates":[{"index":1,"sense_key":"overhaul%1:04:00::","synset_id":"overhaul.n.01"},{"index":2,"sense_key":"overhaul%1:04:01::","synset_id":"renovation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"overhaul%1:04:01::","call_ids":["semeval2013.d009.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"overhaul%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.00457,"input_uncached_usd":0.0022600000000000003,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.909353749942966}
{"item_id":"semeval2013.d009.s003.t006","gold_sense_keys":["congress%1:14:01::"],"candidates":[{"index":1,"sense_key":"congress%1:04:01::","synset_id":"sexual_intercourse.n.01"},{"index":2,"sense_key":"congress%1:14:00::","synset_id":"congress.n.02"},{"index":3,"sense_key":"congress%1:14:02::","synset_id":"congress.n.03"},{"index":4,"sense_key":"congress%1:14:01::","synset_id":"congress.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"congress%1:14:01::","call_ids":["semeval2013.d009.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"congress%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.0045000000000000005,"input_uncached_usd":0.0024600000000000004,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.512911125086248}
{"item_id":"semeval2013.d009.s003.t007","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:14:00::","synset_id":"class.n.06"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:28:01::","synset_id":"year.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d009.s003.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":540,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00459,"input_uncached_usd":0.0027,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.222742624930106}
{"item_id":"semeval2013.d009.s003.t008","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":2,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":3,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":4,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":5,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":6,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":7,"sense_key":"law%1:14:01::","synset_id":"police.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s003.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.005535,"input_uncached_usd":0.0033150000000000002,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9252357910154387}
{"item_id":"semeval2013.d009.s003.t009","gold_sense_keys":["rallying_cry%1:10:00::"],"candidates":[{"index":1,"sense_key":"rallying_cry%1:10:00::","synset_id":"war_cry.n.01"},{"index":2,"sense_key":"rallying_cry%1:10:01::","synset_id":"war_cry.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rallying_cry%1:10:00::","call_ids":["semeval2013.d009.s003.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rallying_cry%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.004285000000000001,"input_uncached_usd":0.002245,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.424638916971162}
{"item_id":"semeval2013.d009.s004.t000","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":2,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":3,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"},{"index":4,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00449,"input_uncached_usd":0.00269,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4782620830228552}
{"item_id":"semeval2013.d009.s004.t001","gold_sense_keys":["suit%1:04:00::"],"candidates":[{"index":1,"sense_key":"suit%1:06:01::","synset_id":"suit.n.06"},{"index":2,"sense_key":"suit%1:10:00::","synset_id":"suit.n.05"},{"index":3,"sense_key":"suit%1:10:01::","synset_id":"courtship.n.01"},{"index":4,"sense_key":"suit%1:06:00::","synset_id":"suit.n.01"},{"index":5,"sense_key":"suit%1:04:00::","synset_id":"lawsuit.n.01"},{"index":6,"sense_key":"suit%1:18:00::","synset_id":"suit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"suit%1:04:00::","call_ids":["semeval2013.d009.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"suit%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.0051400000000000005,"input_uncached_usd":0.0034000000000000002,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6633518330054358}
{"item_id":"semeval2013.d009.s004.t002","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"},{"index":2,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"},{"index":3,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"},{"index":4,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"hour%1:28:00::","call_ids":["semeval2013.d009.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":542,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0046,"input_uncached_usd":0.00271,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5203998329816386}
{"item_id":"semeval2013.d009.s004.t003","gold_sense_keys":["argument%1:10:00::"],"candidates":[{"index":1,"sense_key":"argument%1:09:01::","synset_id":"argumentation.n.02"},{"index":2,"sense_key":"argument%1:10:04::","synset_id":"argument.n.05"},{"index":3,"sense_key":"argument%1:10:00::","synset_id":"argument.n.03"},{"index":4,"sense_key":"argument%1:10:01::","synset_id":"argument.n.04"},{"index":5,"sense_key":"argument%1:10:03::","synset_id":"controversy.n.01"},{"index":6,"sense_key":"argument%1:09:00::","synset_id":"argument.n.06"},{"index":7,"sense_key":"argument%1:10:02::","synset_id":"argument.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"argument%1:10:00::","call_ids":["semeval2013.d009.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"argument%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":114},"cost":{"currency":"USD","total_usd":0.007550000000000001,"input_uncached_usd":0.0036500000000000005,"input_cached_usd":0.0,"output_usd":0.0039000000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.638611249974929}
{"item_id":"semeval2013.d009.s004.t004","gold_sense_keys":["march%1:28:00::"],"candidates":[{"index":1,"sense_key":"march%1:10:01::","synset_id":"marching_music.n.01"},{"index":2,"sense_key":"march%1:10:00::","synset_id":"master_of_architecture.n.01"},{"index":3,"sense_key":"march%1:04:02::","synset_id":"march.n.03"},{"index":4,"sense_key":"march%1:04:00::","synset_id":"march.n.02"},{"index":5,"sense_key":"march%1:15:00::","synset_id":"borderland.n.01"},{"index":6,"sense_key":"march%1:14:00::","synset_id":"march.n.04"},{"index":7,"sense_key":"march%1:28:00::","synset_id":"march.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"march%1:28:00::","call_ids":["semeval2013.d009.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"march%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":656,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.00487,"input_uncached_usd":0.0032800000000000004,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.788602582993917}
{"item_id":"semeval2013.d009.s004.t006","gold_sense_keys":["race%1:11:01::"],"candidates":[{"index":1,"sense_key":"race%1:14:01::","synset_id":"subspecies.n.01"},{"index":2,"sense_key":"race%1:06:00::","synset_id":"raceway.n.01"},{"index":3,"sense_key":"race%1:14:00::","synset_id":"race.n.03"},{"index":4,"sense_key":"race%1:11:01::","synset_id":"race.n.01"},{"index":5,"sense_key":"race%1:11:00::","synset_id":"race.n.02"},{"index":6,"sense_key":"race%1:19:00::","synset_id":"slipstream.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"race%1:11:01::","call_ids":["semeval2013.d009.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"race%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.004765,"input_uncached_usd":0.0030250000000000003,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.957867291988805}
{"item_id":"semeval2013.d009.s004.t007","gold_sense_keys":["swing%1:26:01::"],"candidates":[{"index":1,"sense_key":"swing%1:04:02::","synset_id":"baseball_swing.n.01"},{"index":2,"sense_key":"swing%1:26:01::","synset_id":"swing.n.01"},{"index":3,"sense_key":"swing%1:04:03::","synset_id":"golf_stroke.n.01"},{"index":4,"sense_key":"swing%1:10:01::","synset_id":"swing.n.05"},{"index":5,"sense_key":"swing%1:06:00::","synset_id":"swing.n.02"},{"index":6,"sense_key":"swing%1:04:04::","synset_id":"swing.n.09"},{"index":7,"sense_key":"swing%1:04:00::","synset_id":"swing.n.04"},{"index":8,"sense_key":"swing%1:07:00::","synset_id":"lilt.n.01"},{"index":9,"sense_key":"swing%1:04:06::","synset_id":"swing.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"swing%1:26:01::","call_ids":["semeval2013.d009.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"swing%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.005860000000000001,"input_uncached_usd":0.0036400000000000004,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7129888749914244}
{"item_id":"semeval2013.d009.s004.t009","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:04:00::","synset_id":"court.n.11"},{"index":2,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":3,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":4,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":7,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":8,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":9,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":10,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":11,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"court%1:14:00::","call_ids":["semeval2013.d009.s004.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":818,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01993,"input_uncached_usd":0.004090000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":18.838637916953303}
{"item_id":"semeval2013.d009.s004.t010","gold_sense_keys":["decision%1:09:00::"],"candidates":[{"index":1,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"},{"index":2,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"},{"index":3,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"},{"index":4,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"},{"index":5,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decision%1:09:00::","call_ids":["semeval2013.d009.s004.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decision%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.00551,"input_uncached_usd":0.00302,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9403873330447823}
{"item_id":"semeval2013.d009.s005.t001","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":2,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":3,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":4,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":5,"sense_key":"court%1:04:00::","synset_id":"court.n.11"},{"index":6,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":7,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":8,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:14:01::","synset_id":"court.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"court%1:14:00::","call_ids":["semeval2013.d009.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":858,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020130000000000002,"input_uncached_usd":0.00429,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.722173833986744}
{"item_id":"semeval2013.d009.s005.t002","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":2,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":3,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":4,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":5,"sense_key":"law%1:14:01::","synset_id":"police.n.01"},{"index":6,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":7,"sense_key":"law%1:04:00::","synset_id":"law.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.006295,"input_uncached_usd":0.0037150000000000004,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0396495839813724}
{"item_id":"semeval2013.d009.s006.t003","gold_sense_keys":["dinner%1:14:00::"],"candidates":[{"index":1,"sense_key":"dinner%1:14:00::","synset_id":"dinner.n.02"},{"index":2,"sense_key":"dinner%1:13:00::","synset_id":"dinner.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dinner%1:14:00::","call_ids":["semeval2013.d009.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dinner%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":95},"cost":{"currency":"USD","total_usd":0.00597,"input_uncached_usd":0.00267,"input_cached_usd":0.0,"output_usd":0.0033,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8175522920209914}
{"item_id":"semeval2013.d009.s007.t001","gold_sense_keys":["dinner%1:14:00::"],"candidates":[{"index":1,"sense_key":"dinner%1:13:00::","synset_id":"dinner.n.01"},{"index":2,"sense_key":"dinner%1:14:00::","synset_id":"dinner.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dinner%1:14:00::","call_ids":["semeval2013.d009.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dinner%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":186,"reasoning_output_tokens":170},"cost":{"currency":"USD","total_usd":0.00803,"input_uncached_usd":0.0024500000000000004,"input_cached_usd":0.0,"output_usd":0.00558,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.981509958044626}
{"item_id":"semeval2013.d009.s007.t002","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":2,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":3,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":4,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":5,"sense_key":"law%1:14:01::","synset_id":"police.n.01"},{"index":6,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":7,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":713,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.005755000000000001,"input_uncached_usd":0.0035650000000000005,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6511134579777718}
{"item_id":"semeval2013.d009.s008.t000","gold_sense_keys":["sponsor%1:18:00::"],"candidates":[{"index":1,"sense_key":"sponsor%1:18:00::","synset_id":"patron.n.03"},{"index":2,"sense_key":"sponsor%1:18:01::","synset_id":"presenter.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sponsor%1:18:00::","call_ids":["semeval2013.d009.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sponsor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":79},"cost":{"currency":"USD","total_usd":0.00513,"input_uncached_usd":0.00231,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.257021041936241}
{"item_id":"semeval2013.d009.s008.t001","gold_sense_keys":["giant%1:14:00::"],"candidates":[{"index":1,"sense_key":"giant%1:18:02::","synset_id":"giant.n.06"},{"index":2,"sense_key":"giant%1:14:00::","synset_id":"giant.n.03"},{"index":3,"sense_key":"giant%1:17:00::","synset_id":"giant_star.n.01"},{"index":4,"sense_key":"giant%1:18:01::","synset_id":"colossus.n.02"},{"index":5,"sense_key":"giant%1:18:00::","synset_id":"giant.n.05"},{"index":6,"sense_key":"giant%1:05:00::","synset_id":"giant.n.01"},{"index":7,"sense_key":"giant%1:18:03::","synset_id":"giant.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"giant%1:14:00::","call_ids":["semeval2013.d009.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"giant%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":635,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.005035,"input_uncached_usd":0.0031750000000000003,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0498728750972077}
{"item_id":"semeval2013.d009.s008.t002","gold_sense_keys":["stake%1:21:02::"],"candidates":[{"index":1,"sense_key":"stake%1:06:01::","synset_id":"stake.n.03"},{"index":2,"sense_key":"stake%1:21:00::","synset_id":"stake.n.04"},{"index":3,"sense_key":"stake%1:06:00::","synset_id":"stake.n.05"},{"index":4,"sense_key":"stake%1:21:02::","synset_id":"interest.n.05"},{"index":5,"sense_key":"stake%1:10:00::","synset_id":"post.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"stake%1:21:02::","call_ids":["semeval2013.d009.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"stake%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.004645,"input_uncached_usd":0.0030250000000000003,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6206672500120476}
{"item_id":"semeval2013.d009.s009.t000","gold_sense_keys":["dinner%1:14:00::"],"candidates":[{"index":1,"sense_key":"dinner%1:14:00::","synset_id":"dinner.n.02"},{"index":2,"sense_key":"dinner%1:13:00::","synset_id":"dinner.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dinner%1:14:00::","call_ids":["semeval2013.d009.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dinner%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.005425,"input_uncached_usd":0.0023650000000000003,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.335934041067958}
{"item_id":"semeval2013.d009.s009.t001","gold_sense_keys":["washington%1:15:01::"],"candidates":[{"index":1,"sense_key":"washington%1:14:00::","synset_id":"capital.n.06"},{"index":2,"sense_key":"washington%1:18:00::","synset_id":"washington.n.04"},{"index":3,"sense_key":"washington%1:18:01::","synset_id":"washington.n.05"},{"index":4,"sense_key":"washington%1:15:00::","synset_id":"washington.n.02"},{"index":5,"sense_key":"washington%1:15:01::","synset_id":"washington.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"washington%1:15:01::","call_ids":["semeval2013.d009.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"washington%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.005840000000000001,"input_uncached_usd":0.0031100000000000004,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0986775420606136}
{"item_id":"semeval2013.d009.s009.t003","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"},{"index":2,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"},{"index":3,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"},{"index":4,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"hour%1:28:00::","call_ids":["semeval2013.d009.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.005615,"input_uncached_usd":0.0026750000000000003,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6182144170161337}
{"item_id":"semeval2013.d009.s009.t004","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:04:00::","synset_id":"court.n.11"},{"index":2,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":3,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":4,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":5,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":6,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":9,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":10,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":11,"sense_key":"court%1:06:04::","synset_id":"court.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"court%1:14:00::","call_ids":["semeval2013.d009.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":2929,"reasoning_output_tokens":2882},"cost":{"currency":"USD","total_usd":0.091925,"input_uncached_usd":0.0040550000000000004,"input_cached_usd":0.0,"output_usd":0.08787,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":42.46643074997701}
{"item_id":"semeval2013.d009.s009.t005","gold_sense_keys":["conference%1:10:00::","conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":2,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"},{"index":3,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"conference%1:14:00::","call_ids":["semeval2013.d009.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":176,"reasoning_output_tokens":160},"cost":{"currency":"USD","total_usd":0.007680000000000001,"input_uncached_usd":0.0024000000000000002,"input_cached_usd":0.0,"output_usd":0.00528,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.925362791051157}
{"item_id":"semeval2013.d009.s009.t006","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":2,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":3,"sense_key":"case%1:06:01::","synset_id":"case.n.20"},{"index":4,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":7,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":8,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":9,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":10,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":11,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":14,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":15,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":16,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":17,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":18,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":19,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":20,"sense_key":"case%1:10:02::","synset_id":"case.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1239,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.008565000000000001,"input_uncached_usd":0.006195000000000001,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9030587080633268}
{"item_id":"semeval2013.d009.s010.t000","gold_sense_keys":["attendance%1:04:00::"],"candidates":[{"index":1,"sense_key":"attendance%1:28:00::","synset_id":"attendance.n.02"},{"index":2,"sense_key":"attendance%1:04:00::","synset_id":"attendance.n.01"},{"index":3,"sense_key":"attendance%1:14:00::","synset_id":"attendance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"attendance%1:04:00::","call_ids":["semeval2013.d009.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"attendance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.00451,"input_uncached_usd":0.0022,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5569817080395296}
{"item_id":"semeval2013.d009.s010.t001","gold_sense_keys":["senate%1:14:01::"],"candidates":[{"index":1,"sense_key":"senate%1:14:01::","synset_id":"united_states_senate.n.01"},{"index":2,"sense_key":"senate%1:14:00::","synset_id":"senate.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"senate%1:14:01::","call_ids":["semeval2013.d009.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"senate%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":394,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.00407,"input_uncached_usd":0.00197,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.07776929193642}
{"item_id":"semeval2013.d009.s010.t002","gold_sense_keys":["opponent%1:18:00::"],"candidates":[{"index":1,"sense_key":"opponent%1:18:02::","synset_id":"opposition.n.04"},{"index":2,"sense_key":"opponent%1:18:00::","synset_id":"adversary.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"opponent%1:18:00::","call_ids":["semeval2013.d009.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"opponent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":401,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.003985000000000001,"input_uncached_usd":0.0020050000000000003,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9401857500197366}
{"item_id":"semeval2013.d009.s011.t000","gold_sense_keys":["guest%1:18:00::"],"candidates":[{"index":1,"sense_key":"guest%1:18:02::","synset_id":"guest.n.02"},{"index":2,"sense_key":"guest%1:06:00::","synset_id":"node.n.08"},{"index":3,"sense_key":"guest%1:18:01::","synset_id":"guest.n.03"},{"index":4,"sense_key":"guest%1:18:00::","synset_id":"guest.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"guest%1:18:00::","call_ids":["semeval2013.d009.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"guest%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.004375,"input_uncached_usd":0.002245,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7698023340199143}
{"item_id":"semeval2013.d009.s011.t001","gold_sense_keys":["dinner%1:14:00::"],"candidates":[{"index":1,"sense_key":"dinner%1:13:00::","synset_id":"dinner.n.01"},{"index":2,"sense_key":"dinner%1:14:00::","synset_id":"dinner.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dinner%1:14:00::","call_ids":["semeval2013.d009.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dinner%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":406,"cached_input_tokens":0,"output_tokens":212,"reasoning_output_tokens":196},"cost":{"currency":"USD","total_usd":0.00839,"input_uncached_usd":0.00203,"input_cached_usd":0.0,"output_usd":0.00636,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.031371916062199}
{"item_id":"semeval2013.d009.s012.t000","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"},{"index":4,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.004095,"input_uncached_usd":0.0021750000000000003,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8907176670618355}
{"item_id":"semeval2013.d009.s012.t001","gold_sense_keys":["event%1:03:00::"],"candidates":[{"index":1,"sense_key":"event%1:19:01::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"event%1:19:00::","synset_id":"event.n.03"},{"index":3,"sense_key":"event%1:26:00::","synset_id":"event.n.02"},{"index":4,"sense_key":"event%1:03:00::","synset_id":"event.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"event%1:03:00::","call_ids":["semeval2013.d009.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"event%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.00372,"input_uncached_usd":0.0022800000000000003,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.801911542075686}
{"item_id":"semeval2013.d009.s012.t002","gold_sense_keys":["years%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":2,"sense_key":"years%1:28:01::","synset_id":"long_time.n.01"},{"index":3,"sense_key":"class%1:14:04::","synset_id":"class.n.06"},{"index":4,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":5,"sense_key":"years%1:28:00::","synset_id":"old_age.n.01"},{"index":6,"sense_key":"years%1:28:02::","synset_id":"days.n.01"},{"index":7,"sense_key":"year%1:28:00::","synset_id":"year.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"years%1:28:01::","call_ids":["semeval2013.d009.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"years%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.00572,"input_uncached_usd":0.0031400000000000004,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8943807909963652}
{"item_id":"semeval2013.d009.s013.t002","gold_sense_keys":["rule%1:10:00::"],"candidates":[{"index":1,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":2,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":3,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":4,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":5,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":6,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":7,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":8,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":9,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":10,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":11,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":12,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"rule%1:10:00::","call_ids":["semeval2013.d009.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"rule%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":826,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":85},"cost":{"currency":"USD","total_usd":0.00716,"input_uncached_usd":0.00413,"input_cached_usd":0.0,"output_usd":0.00303,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2064384579425678}
{"item_id":"semeval2013.d009.s014.t001","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.004665000000000001,"input_uncached_usd":0.0023250000000000002,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5792751669650897}
{"item_id":"semeval2013.d009.s014.t003","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:00::","synset_id":"action.n.10"},{"index":2,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":3,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":4,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":8,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":9,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":10,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d009.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":769,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.006665000000000001,"input_uncached_usd":0.0038450000000000003,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5228195830713958}
{"item_id":"semeval2013.d009.s014.t004","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":3,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":4,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.00493,"input_uncached_usd":0.00232,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9675772920018062}
{"item_id":"semeval2013.d009.s015.t000","gold_sense_keys":["code%1:10:01::"],"candidates":[{"index":1,"sense_key":"code%1:10:01::","synset_id":"code.n.01"},{"index":2,"sense_key":"code%1:10:02::","synset_id":"code.n.03"},{"index":3,"sense_key":"code%1:10:00::","synset_id":"code.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"code%1:10:01::","call_ids":["semeval2013.d009.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"code%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.004385,"input_uncached_usd":0.0021950000000000003,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4265704589197412}
{"item_id":"semeval2013.d009.s015.t002","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d009.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.00426,"input_uncached_usd":0.00201,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.514396750018932}
{"item_id":"semeval2013.d009.s015.t003","gold_sense_keys":["event%1:03:00::"],"candidates":[{"index":1,"sense_key":"event%1:26:00::","synset_id":"event.n.02"},{"index":2,"sense_key":"event%1:03:00::","synset_id":"event.n.01"},{"index":3,"sense_key":"event%1:19:01::","synset_id":"consequence.n.01"},{"index":4,"sense_key":"event%1:19:00::","synset_id":"event.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"event%1:03:00::","call_ids":["semeval2013.d009.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"event%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":81},"cost":{"currency":"USD","total_usd":0.005455,"input_uncached_usd":0.0025450000000000004,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.243008208926767}
{"item_id":"semeval2013.d009.s015.t004","gold_sense_keys":["organization%1:14:00::","organization%1:14:01::"],"candidates":[{"index":1,"sense_key":"organization%1:04:02::","synset_id":"constitution.n.02"},{"index":2,"sense_key":"organization%1:04:00::","synset_id":"organization.n.04"},{"index":3,"sense_key":"organization%1:14:00::","synset_id":"organization.n.01"},{"index":4,"sense_key":"organization%1:04:01::","synset_id":"organization.n.06"},{"index":5,"sense_key":"organization%1:07:00::","synset_id":"organization.n.05"},{"index":6,"sense_key":"organization%1:09:00::","synset_id":"arrangement.n.03"},{"index":7,"sense_key":"organization%1:14:01::","synset_id":"administration.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"organization%1:14:00::","call_ids":["semeval2013.d009.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"organization%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.005925000000000001,"input_uncached_usd":0.0035550000000000004,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6035486250184476}
{"item_id":"semeval2013.d009.s015.t005","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d009.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.004200000000000001,"input_uncached_usd":0.00201,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6736738750478253}
{"item_id":"semeval2013.d009.s015.t006","gold_sense_keys":["speaker%1:18:00::"],"candidates":[{"index":1,"sense_key":"speaker%1:18:01::","synset_id":"speaker.n.03"},{"index":2,"sense_key":"speaker%1:18:00::","synset_id":"speaker.n.01"},{"index":3,"sense_key":"speaker%1:06:00::","synset_id":"loudspeaker.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"speaker%1:18:00::","call_ids":["semeval2013.d009.s015.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"speaker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.00453,"input_uncached_usd":0.00249,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.933846958912909}
{"item_id":"semeval2013.d009.s015.t008","gold_sense_keys":["program%1:10:00::"],"candidates":[{"index":1,"sense_key":"program%1:10:01::","synset_id":"broadcast.n.02"},{"index":2,"sense_key":"program%1:10:02::","synset_id":"program.n.07"},{"index":3,"sense_key":"program%1:10:05::","synset_id":"course_of_study.n.01"},{"index":4,"sense_key":"program%1:09:00::","synset_id":"plan.n.01"},{"index":5,"sense_key":"program%1:04:00::","synset_id":"program.n.08"},{"index":6,"sense_key":"program%1:10:04::","synset_id":"platform.n.02"},{"index":7,"sense_key":"program%1:09:01::","synset_id":"program.n.02"},{"index":8,"sense_key":"program%1:10:00::","synset_id":"program.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"program%1:04:00::","call_ids":["semeval2013.d009.s015.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"program%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":727,"cached_input_tokens":0,"output_tokens":1432,"reasoning_output_tokens":1404},"cost":{"currency":"USD","total_usd":0.046595,"input_uncached_usd":0.003635,"input_cached_usd":0.0,"output_usd":0.04296,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":26.20849495800212}
{"item_id":"semeval2013.d009.s015.t009","gold_sense_keys":["event%1:03:00::"],"candidates":[{"index":1,"sense_key":"event%1:03:00::","synset_id":"event.n.01"},{"index":2,"sense_key":"event%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"event%1:19:01::","synset_id":"consequence.n.01"},{"index":4,"sense_key":"event%1:19:00::","synset_id":"event.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"event%1:03:00::","call_ids":["semeval2013.d009.s015.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"event%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.004435000000000001,"input_uncached_usd":0.0025450000000000004,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.411499417037703}
{"item_id":"semeval2013.d009.s016.t000","gold_sense_keys":["proximity%1:07:00::"],"candidates":[{"index":1,"sense_key":"proximity%1:07:00::","synset_id":"proximity.n.01"},{"index":2,"sense_key":"proximity%1:09:00::","synset_id":"proximity.n.03"},{"index":3,"sense_key":"proximity%1:15:00::","synset_id":"proximity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"proximity%1:07:00::","call_ids":["semeval2013.d009.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"proximity%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":80},"cost":{"currency":"USD","total_usd":0.00521,"input_uncached_usd":0.00236,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3599408339941874}
{"item_id":"semeval2013.d009.s016.t002","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":2,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":3,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":4,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":5,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":6,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":7,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":8,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":9,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":10,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":11,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":12,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":13,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":14,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":15,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":16,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":17,"sense_key":"case%1:06:01::","synset_id":"case.n.20"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":20,"sense_key":"case%1:06:02::","synset_id":"case.n.19"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1216,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.00839,"input_uncached_usd":0.00608,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.404197500087321}
{"item_id":"semeval2013.d009.s016.t003","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":2,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":3,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":4,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":5,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":6,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":7,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":8,"sense_key":"company%1:14:05::","synset_id":"company.n.09"},{"index":9,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d009.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.005425,"input_uncached_usd":0.0035350000000000004,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.896949875052087}
{"item_id":"semeval2013.d009.s016.t004","gold_sense_keys":["interest%1:21:03::"],"candidates":[{"index":1,"sense_key":"interest%1:04:01::","synset_id":"pastime.n.01"},{"index":2,"sense_key":"interest%1:09:00::","synset_id":"interest.n.01"},{"index":3,"sense_key":"interest%1:21:00::","synset_id":"interest.n.04"},{"index":4,"sense_key":"interest%1:07:02::","synset_id":"interest.n.03"},{"index":5,"sense_key":"interest%1:07:01::","synset_id":"sake.n.01"},{"index":6,"sense_key":"interest%1:14:00::","synset_id":"interest.n.06"},{"index":7,"sense_key":"interest%1:21:03::","synset_id":"interest.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"interest%1:21:03::","call_ids":["semeval2013.d009.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"interest%1:21:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":683,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.005785,"input_uncached_usd":0.003415,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6081454580416903}
{"item_id":"semeval2013.d009.s017.t000","gold_sense_keys":["breach%1:04:01::"],"candidates":[{"index":1,"sense_key":"breach%1:04:01::","synset_id":"breach.n.01"},{"index":2,"sense_key":"breach%1:11:00::","synset_id":"rupture.n.02"},{"index":3,"sense_key":"breach%1:17:00::","synset_id":"breach.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"breach%1:04:01::","call_ids":["semeval2013.d009.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"breach%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":88},"cost":{"currency":"USD","total_usd":0.005525,"input_uncached_usd":0.002405,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.089581499923952}
{"item_id":"semeval2013.d009.s017.t002","gold_sense_keys":["indifference%1:07:01::"],"candidates":[{"index":1,"sense_key":"indifference%1:12:01::","synset_id":"emotionlessness.n.01"},{"index":2,"sense_key":"indifference%1:12:00::","synset_id":"indifference.n.01"},{"index":3,"sense_key":"indifference%1:07:01::","synset_id":"nonchalance.n.01"},{"index":4,"sense_key":"indifference%1:07:00::","synset_id":"apathy.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"indifference%1:07:01::","call_ids":["semeval2013.d009.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"indifference%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":137},"cost":{"currency":"USD","total_usd":0.0072250000000000005,"input_uncached_usd":0.002635,"input_cached_usd":0.0,"output_usd":0.00459,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3435020829783753}
{"item_id":"semeval2013.d009.s017.t003","gold_sense_keys":["code%1:10:01::"],"candidates":[{"index":1,"sense_key":"code%1:10:02::","synset_id":"code.n.03"},{"index":2,"sense_key":"code%1:10:00::","synset_id":"code.n.02"},{"index":3,"sense_key":"code%1:10:01::","synset_id":"code.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"code%1:10:01::","call_ids":["semeval2013.d009.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"code%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":41,"reasoning_output_tokens":26},"cost":{"currency":"USD","total_usd":0.0035900000000000003,"input_uncached_usd":0.00236,"input_cached_usd":0.0,"output_usd":0.00123,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4957931250100955}
{"item_id":"semeval2013.d009.s017.t004","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":2,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"},{"index":3,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":4,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"claim%1:10:02::","call_ids":["semeval2013.d009.s017.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"claim%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.005035,"input_uncached_usd":0.0030250000000000003,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5415537090739235}
{"item_id":"semeval2013.d009.s017.t005","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"},{"index":2,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004405,"input_uncached_usd":0.002605,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4690279169008136}
{"item_id":"semeval2013.d009.s017.t006","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":2,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":3,"sense_key":"court%1:04:00::","synset_id":"court.n.11"},{"index":4,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":7,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":8,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":9,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":10,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":11,"sense_key":"court%1:06:00::","synset_id":"court.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"court%1:14:00::","call_ids":["semeval2013.d009.s017.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":801,"cached_input_tokens":0,"output_tokens":262,"reasoning_output_tokens":246},"cost":{"currency":"USD","total_usd":0.011865,"input_uncached_usd":0.004005000000000001,"input_cached_usd":0.0,"output_usd":0.00786,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.242410916951485}
{"item_id":"semeval2013.d009.s017.t007","gold_sense_keys":["rule%1:10:00::"],"candidates":[{"index":1,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":2,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":3,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":4,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":5,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":6,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":7,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":8,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":9,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":10,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"},{"index":11,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":12,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"rule%1:10:00::","call_ids":["semeval2013.d009.s017.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"rule%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":919,"cached_input_tokens":0,"output_tokens":360,"reasoning_output_tokens":344},"cost":{"currency":"USD","total_usd":0.015395,"input_uncached_usd":0.004595,"input_cached_usd":0.0,"output_usd":0.0108,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.296424707979895}
{"item_id":"semeval2013.d009.s017.t008","gold_sense_keys":["president%1:18:01::"],"candidates":[{"index":1,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":2,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":3,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":4,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":5,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"},{"index":6,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"president%1:18:03::","call_ids":["semeval2013.d009.s017.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"president%1:18:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":608,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01888,"input_uncached_usd":0.00304,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.320539250038564}
{"item_id":"semeval2013.d009.s018.t000","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"},{"index":2,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.00421,"input_uncached_usd":0.0026200000000000004,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7895262090023607}
{"item_id":"semeval2013.d009.s018.t001","gold_sense_keys":["fundraiser%1:11:00::"],"candidates":[{"index":1,"sense_key":"fundraiser%1:11:00::","synset_id":"fundraiser.n.02"},{"index":2,"sense_key":"fundraiser%1:18:00::","synset_id":"fundraiser.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fundraiser%1:11:00::","call_ids":["semeval2013.d009.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fundraiser%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":425,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.003535,"input_uncached_usd":0.002125,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.563343874993734}
{"item_id":"semeval2013.d009.s018.t003","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":2,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":3,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":4,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":5,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":6,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":7,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":8,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":9,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":10,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":11,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":12,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":13,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":14,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":15,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":16,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":17,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":18,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":19,"sense_key":"case%1:06:01::","synset_id":"case.n.20"},{"index":20,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1233,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.008175,"input_uncached_usd":0.006165,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.206220666063018}
{"item_id":"semeval2013.d009.s018.t004","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":3,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":4,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":5,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":6,"sense_key":"court%1:04:00::","synset_id":"court.n.11"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":9,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":10,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":11,"sense_key":"court%1:06:05::","synset_id":"court.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"court%1:14:00::","call_ids":["semeval2013.d009.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":804,"cached_input_tokens":0,"output_tokens":4086,"reasoning_output_tokens":4027},"cost":{"currency":"USD","total_usd":0.1266,"input_uncached_usd":0.00402,"input_cached_usd":0.0,"output_usd":0.12258000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":61.54935845802538}
{"item_id":"semeval2013.d009.s019.t000","gold_sense_keys":["appearance%1:04:00::"],"candidates":[{"index":1,"sense_key":"appearance%1:07:00::","synset_id":"appearance.n.01"},{"index":2,"sense_key":"appearance%1:04:02::","synset_id":"appearance.n.06"},{"index":3,"sense_key":"appearance%1:04:00::","synset_id":"appearance.n.05"},{"index":4,"sense_key":"appearance%1:04:01::","synset_id":"appearance.n.03"},{"index":5,"sense_key":"appearance%1:09:00::","synset_id":"appearance.n.04"},{"index":6,"sense_key":"appearance%1:11:00::","synset_id":"appearance.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"appearance%1:04:00::","call_ids":["semeval2013.d009.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"appearance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":73},"cost":{"currency":"USD","total_usd":0.005730000000000001,"input_uncached_usd":0.0030900000000000003,"input_cached_usd":0.0,"output_usd":0.00264,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9927959170890972}
{"item_id":"semeval2013.d009.s019.t001","gold_sense_keys":["assistance%1:04:00::"],"candidates":[{"index":1,"sense_key":"assistance%1:04:00::","synset_id":"aid.n.02"},{"index":2,"sense_key":"assistance%1:07:00::","synset_id":"aid.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assistance%1:04:00::","call_ids":["semeval2013.d009.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assistance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.004295,"input_uncached_usd":0.002435,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7872256669215858}
{"item_id":"semeval2013.d009.s019.t002","gold_sense_keys":["event%1:03:00::"],"candidates":[{"index":1,"sense_key":"event%1:26:00::","synset_id":"event.n.02"},{"index":2,"sense_key":"event%1:03:00::","synset_id":"event.n.01"},{"index":3,"sense_key":"event%1:19:01::","synset_id":"consequence.n.01"},{"index":4,"sense_key":"event%1:19:00::","synset_id":"event.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"event%1:03:00::","call_ids":["semeval2013.d009.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"event%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":566,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.0052,"input_uncached_usd":0.00283,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4875485410448164}
{"item_id":"semeval2013.d009.s019.t003","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":2,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":3,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"},{"index":4,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":5,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":6,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"claim%1:10:02::","call_ids":["semeval2013.d009.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"claim%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":79},"cost":{"currency":"USD","total_usd":0.005995,"input_uncached_usd":0.0031450000000000002,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2138511249795556}
{"item_id":"semeval2013.d009.s020.t000","gold_sense_keys":["regard%1:09:01::"],"candidates":[{"index":1,"sense_key":"regard%1:10:00::","synset_id":"regard.n.03"},{"index":2,"sense_key":"regard%1:26:00::","synset_id":"esteem.n.01"},{"index":3,"sense_key":"regard%1:12:02::","synset_id":"regard.n.06"},{"index":4,"sense_key":"regard%1:09:01::","synset_id":"attentiveness.n.01"},{"index":5,"sense_key":"regard%1:04:00::","synset_id":"gaze.n.01"},{"index":6,"sense_key":"regard%1:09:02::","synset_id":"respect.n.03"},{"index":7,"sense_key":"regard%1:09:00::","synset_id":"respect.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"regard%1:09:01::","call_ids":["semeval2013.d009.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"regard%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01935,"input_uncached_usd":0.00351,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.521154582966119}
{"item_id":"semeval2013.d009.s020.t002","gold_sense_keys":["business%1:09:01::"],"candidates":[{"index":1,"sense_key":"business%1:14:02::","synset_id":"business.n.07"},{"index":2,"sense_key":"business%1:04:02::","synset_id":"business.n.06"},{"index":3,"sense_key":"business%1:14:01::","synset_id":"clientele.n.01"},{"index":4,"sense_key":"business%1:09:00::","synset_id":"business.n.05"},{"index":5,"sense_key":"business%1:04:03::","synset_id":"business.n.09"},{"index":6,"sense_key":"business%1:04:00::","synset_id":"occupation.n.01"},{"index":7,"sense_key":"business%1:14:00::","synset_id":"business.n.01"},{"index":8,"sense_key":"business%1:09:01::","synset_id":"business.n.04"},{"index":9,"sense_key":"business%1:04:01::","synset_id":"commercial_enterprise.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"business%1:09:01::","call_ids":["semeval2013.d009.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"business%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":294,"reasoning_output_tokens":278},"cost":{"currency":"USD","total_usd":0.012625,"input_uncached_usd":0.0038050000000000002,"input_cached_usd":0.0,"output_usd":0.00882,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.855303875054233}
{"item_id":"semeval2013.d009.s020.t003","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":2,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":3,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":6,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":7,"sense_key":"court%1:04:00::","synset_id":"court.n.11"},{"index":8,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":9,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":10,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":11,"sense_key":"court%1:14:01::","synset_id":"court.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"court%1:14:00::","call_ids":["semeval2013.d009.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":818,"cached_input_tokens":0,"output_tokens":3034,"reasoning_output_tokens":2989},"cost":{"currency":"USD","total_usd":0.09511,"input_uncached_usd":0.004090000000000001,"input_cached_usd":0.0,"output_usd":0.09102,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":48.57833741698414}
{"item_id":"semeval2013.d009.s020.t004","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["semeval2013.d009.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004285000000000001,"input_uncached_usd":0.0025150000000000003,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8259133329847828}
{"item_id":"semeval2013.d009.s022.t001","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":4,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.00402,"input_uncached_usd":0.0024300000000000003,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6142047500470653}
{"item_id":"semeval2013.d009.s022.t002","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":2,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":3,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"},{"index":4,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":5,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":6,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":7,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":8,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":11,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"issue%1:09:01::","call_ids":["semeval2013.d009.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":876,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":117},"cost":{"currency":"USD","total_usd":0.00837,"input_uncached_usd":0.00438,"input_cached_usd":0.0,"output_usd":0.0039900000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7083864170126617}
{"item_id":"semeval2013.d009.s023.t001","gold_sense_keys":["administration%1:28:00::"],"candidates":[{"index":1,"sense_key":"administration%1:04:04::","synset_id":"government.n.02"},{"index":2,"sense_key":"administration%1:04:01::","synset_id":"administration.n.03"},{"index":3,"sense_key":"administration%1:04:02::","synset_id":"administration.n.06"},{"index":4,"sense_key":"administration%1:28:00::","synset_id":"presidency.n.01"},{"index":5,"sense_key":"administration%1:04:00::","synset_id":"administration.n.01"},{"index":6,"sense_key":"administration%1:14:00::","synset_id":"administration.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"administration%1:14:00::","call_ids":["semeval2013.d009.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"administration%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01885,"input_uncached_usd":0.00301,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.97241137502715}
{"item_id":"semeval2013.d009.s023.t003","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":2,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":3,"sense_key":"law%1:14:01::","synset_id":"police.n.01"},{"index":4,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":5,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":6,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":7,"sense_key":"law%1:09:00::","synset_id":"law.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.00595,"input_uncached_usd":0.00319,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3124457080848515}
{"item_id":"semeval2013.d009.s023.t005","gold_sense_keys":["level%1:26:01::"],"candidates":[{"index":1,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":2,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"},{"index":3,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":4,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":5,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"level%1:26:00::","call_ids":["semeval2013.d009.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"level%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":512,"reasoning_output_tokens":496},"cost":{"currency":"USD","total_usd":0.018765,"input_uncached_usd":0.0034050000000000005,"input_cached_usd":0.0,"output_usd":0.01536,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.010672292090021}
{"item_id":"semeval2013.d009.s024.t001","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":2,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":3,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":4,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":5,"sense_key":"case%1:06:01::","synset_id":"case.n.20"},{"index":6,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":7,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":8,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":9,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":10,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":11,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":12,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":13,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":14,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":15,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":16,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":17,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":18,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":19,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":20,"sense_key":"case%1:18:03::","synset_id":"case.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1168,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.00791,"input_uncached_usd":0.005840000000000001,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9057038750033826}
{"item_id":"semeval2013.d009.s024.t002","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":2,"sense_key":"law%1:14:01::","synset_id":"police.n.01"},{"index":3,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":4,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":5,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":6,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":7,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":90},"cost":{"currency":"USD","total_usd":0.006305,"input_uncached_usd":0.003125,"input_cached_usd":0.0,"output_usd":0.00318,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.340133707970381}
{"item_id":"semeval2013.d009.s025.t000","gold_sense_keys":["indication%1:10:00::"],"candidates":[{"index":1,"sense_key":"indication%1:10:01::","synset_id":"indication.n.02"},{"index":2,"sense_key":"indication%1:09:00::","synset_id":"reading.n.03"},{"index":3,"sense_key":"indication%1:10:03::","synset_id":"indication.n.04"},{"index":4,"sense_key":"indication%1:10:02::","synset_id":"indication.n.03"},{"index":5,"sense_key":"indication%1:10:00::","synset_id":"indication.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"indication%1:10:00::","call_ids":["semeval2013.d009.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"indication%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":107},"cost":{"currency":"USD","total_usd":0.00627,"input_uncached_usd":0.0025800000000000003,"input_cached_usd":0.0,"output_usd":0.00369,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.09815937501844}
{"item_id":"semeval2013.d010.s000.t000","gold_sense_keys":["familiarity%1:09:00::"],"candidates":[{"index":1,"sense_key":"familiarity%1:07:03::","synset_id":"casualness.n.01"},{"index":2,"sense_key":"familiarity%1:04:00::","synset_id":"familiarity.n.05"},{"index":3,"sense_key":"familiarity%1:07:02::","synset_id":"familiarity.n.03"},{"index":4,"sense_key":"familiarity%1:09:00::","synset_id":"acquaintance.n.01"},{"index":5,"sense_key":"familiarity%1:07:01::","synset_id":"familiarity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"familiarity%1:09:00::","call_ids":["semeval2013.d010.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"familiarity%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":355,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017615,"input_uncached_usd":0.001775,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.98739495803602}
{"item_id":"semeval2013.d010.s001.t000","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":2,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":3,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":4,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d010.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":366,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.0035400000000000006,"input_uncached_usd":0.0018300000000000002,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7479471249971539}
{"item_id":"semeval2013.d010.s001.t002","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":2,"sense_key":"game%1:04:04::","synset_id":"game.n.11"},{"index":3,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":4,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":5,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":6,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":7,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":8,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:00::","synset_id":"game.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":104},"cost":{"currency":"USD","total_usd":0.00669,"input_uncached_usd":0.0030900000000000003,"input_cached_usd":0.0,"output_usd":0.0036,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1320077079581097}
{"item_id":"semeval2013.d010.s002.t000","gold_sense_keys":["side%1:14:00::"],"candidates":[{"index":1,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":2,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":3,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":4,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":5,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":6,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":7,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":8,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":9,"sense_key":"side%1:11:00::","synset_id":"english.n.04"},{"index":10,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":11,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":12,"sense_key":"side%1:08:00::","synset_id":"side.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"side%1:14:00::","call_ids":["semeval2013.d010.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"side%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":740,"cached_input_tokens":0,"output_tokens":143,"reasoning_output_tokens":127},"cost":{"currency":"USD","total_usd":0.00799,"input_uncached_usd":0.0037,"input_cached_usd":0.0,"output_usd":0.00429,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9360263330163434}
{"item_id":"semeval2013.d010.s002.t002","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":2,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":3,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":4,"sense_key":"player%1:18:03::","synset_id":"player.n.05"},{"index":5,"sense_key":"player%1:18:01::","synset_id":"player.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d010.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.00345,"input_uncached_usd":0.00201,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.115868916036561}
{"item_id":"semeval2013.d010.s002.t003","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":2,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":3,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":4,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":5,"sense_key":"game%1:04:04::","synset_id":"game.n.11"},{"index":6,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":7,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":8,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":9,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":10,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":11,"sense_key":"game%1:13:00::","synset_id":"game.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.0067800000000000004,"input_uncached_usd":0.0032700000000000003,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6758884580340236}
{"item_id":"semeval2013.d010.s003.t002","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":3,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":4,"sense_key":"player%1:18:03::","synset_id":"player.n.05"},{"index":5,"sense_key":"player%1:18:04::","synset_id":"player.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d010.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.0037500000000000003,"input_uncached_usd":0.0022500000000000003,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4370252909138799}
{"item_id":"semeval2013.d010.s003.t004","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":2,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":3,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":7,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":8,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:04::","synset_id":"game.n.11"},{"index":11,"sense_key":"game%1:13:00::","synset_id":"game.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":703,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":80},"cost":{"currency":"USD","total_usd":0.0063950000000000005,"input_uncached_usd":0.0035150000000000003,"input_cached_usd":0.0,"output_usd":0.00288,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3700697500025854}
{"item_id":"semeval2013.d010.s003.t005","gold_sense_keys":["today%1:28:01::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"today%1:28:01::","call_ids":["semeval2013.d010.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"today%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":356,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.00364,"input_uncached_usd":0.0017800000000000001,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.953234750078991}
{"item_id":"semeval2013.d010.s003.t006","gold_sense_keys":["san_jose%1:15:00::"],"candidates":[{"index":1,"sense_key":"san_jose%1:15:00::","synset_id":"san_jose.n.02"},{"index":2,"sense_key":"san_jose%1:15:01::","synset_id":"san_jose.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"san_jose%1:15:00::","call_ids":["semeval2013.d010.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"san_jose%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":361,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.0036650000000000003,"input_uncached_usd":0.0018050000000000002,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8556099999696016}
{"item_id":"semeval2013.d010.s004.t001","gold_sense_keys":["number_one%1:24:00::"],"candidates":[{"index":1,"sense_key":"number_one%1:23:00::","synset_id":"first.n.02"},{"index":2,"sense_key":"number_one%1:24:00::","synset_id":"first.n.01"},{"index":3,"sense_key":"number_one%1:18:00::","synset_id":"number_one.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"number_one%1:24:00::","call_ids":["semeval2013.d010.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"number_one%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.0052,"input_uncached_usd":0.0021100000000000003,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3563683750107884}
{"item_id":"semeval2013.d010.s004.t003","gold_sense_keys":["record%1:04:01::"],"candidates":[{"index":1,"sense_key":"record%1:10:03::","synset_id":"record.n.01"},{"index":2,"sense_key":"record%1:06:00::","synset_id":"phonograph_record.n.01"},{"index":3,"sense_key":"record%1:04:00::","synset_id":"record.n.06"},{"index":4,"sense_key":"record%1:21:00::","synset_id":"record.n.07"},{"index":5,"sense_key":"record%1:04:01::","synset_id":"record.n.04"},{"index":6,"sense_key":"record%1:23:00::","synset_id":"record.n.03"},{"index":7,"sense_key":"record%1:10:02::","synset_id":"record.n.05"},{"index":8,"sense_key":"record%1:10:00::","synset_id":"criminal_record.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"record%1:04:01::","call_ids":["semeval2013.d010.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"record%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":715,"cached_input_tokens":0,"output_tokens":376,"reasoning_output_tokens":360},"cost":{"currency":"USD","total_usd":0.014855,"input_uncached_usd":0.0035750000000000005,"input_cached_usd":0.0,"output_usd":0.01128,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.633623166941106}
{"item_id":"semeval2013.d010.s004.t005","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":2,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":3,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":4,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":5,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":6,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:04:04::","synset_id":"game.n.11"},{"index":9,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":10,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":11,"sense_key":"game%1:04:02::","synset_id":"game.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":722,"cached_input_tokens":0,"output_tokens":192,"reasoning_output_tokens":177},"cost":{"currency":"USD","total_usd":0.00937,"input_uncached_usd":0.0036100000000000004,"input_cached_usd":0.0,"output_usd":0.00576,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.048894708044827}
{"item_id":"semeval2013.d010.s005.t000","gold_sense_keys":["number%1:07:00::"],"candidates":[{"index":1,"sense_key":"number%1:10:02::","synset_id":"number.n.08"},{"index":2,"sense_key":"number%1:07:00::","synset_id":"number.n.01"},{"index":3,"sense_key":"number%1:14:00::","synset_id":"number.n.07"},{"index":4,"sense_key":"number%1:10:05::","synset_id":"number.n.10"},{"index":5,"sense_key":"number%1:23:00::","synset_id":"number.n.02"},{"index":6,"sense_key":"number%1:06:00::","synset_id":"number.n.11"},{"index":7,"sense_key":"number%1:10:04::","synset_id":"act.n.04"},{"index":8,"sense_key":"number%1:10:01::","synset_id":"numeral.n.01"},{"index":9,"sense_key":"number%1:10:03::","synset_id":"phone_number.n.01"},{"index":10,"sense_key":"number%1:10:00::","synset_id":"issue.n.02"},{"index":11,"sense_key":"number%1:07:01::","synset_id":"number.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"number%1:07:00::","call_ids":["semeval2013.d010.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"number%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":830,"cached_input_tokens":0,"output_tokens":380,"reasoning_output_tokens":364},"cost":{"currency":"USD","total_usd":0.015550000000000001,"input_uncached_usd":0.00415,"input_cached_usd":0.0,"output_usd":0.0114,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.006143957958557}
{"item_id":"semeval2013.d010.s005.t002","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":2,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":3,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":4,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":5,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":6,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":7,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":8,"sense_key":"game%1:04:04::","synset_id":"game.n.11"},{"index":9,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":10,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":11,"sense_key":"game%1:05:00::","synset_id":"game.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":769,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.006815,"input_uncached_usd":0.0038450000000000003,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9342568340944126}
{"item_id":"semeval2013.d010.s006.t001","gold_sense_keys":["league%1:14:01::"],"candidates":[{"index":1,"sense_key":"league%1:23:00::","synset_id":"league.n.03"},{"index":2,"sense_key":"league%1:14:00::","synset_id":"league.n.02"},{"index":3,"sense_key":"league%1:14:01::","synset_id":"league.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"league%1:14:01::","call_ids":["semeval2013.d010.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"league%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.00481,"input_uncached_usd":0.0022,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.315105790970847}
{"item_id":"semeval2013.d010.s006.t004","gold_sense_keys":["year%1:28:01::","year%1:28:02::"],"candidates":[{"index":1,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d010.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":105},"cost":{"currency":"USD","total_usd":0.00636,"input_uncached_usd":0.0027300000000000002,"input_cached_usd":0.0,"output_usd":0.00363,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.777155625051819}
{"item_id":"semeval2013.d010.s006.t005","gold_sense_keys":["league%1:14:01::"],"candidates":[{"index":1,"sense_key":"league%1:14:00::","synset_id":"league.n.02"},{"index":2,"sense_key":"league%1:23:00::","synset_id":"league.n.03"},{"index":3,"sense_key":"league%1:14:01::","synset_id":"league.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"league%1:14:01::","call_ids":["semeval2013.d010.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"league%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.0047799999999999995,"input_uncached_usd":0.0022,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.995972333010286}
{"item_id":"semeval2013.d010.s006.t007","gold_sense_keys":["competition%1:11:00::"],"candidates":[{"index":1,"sense_key":"competition%1:18:00::","synset_id":"rival.n.01"},{"index":2,"sense_key":"competition%1:04:00::","synset_id":"competition.n.03"},{"index":3,"sense_key":"competition%1:11:00::","synset_id":"contest.n.01"},{"index":4,"sense_key":"competition%1:24:01::","synset_id":"competition.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"competition%1:11:00::","call_ids":["semeval2013.d010.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"competition%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":523,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.005825,"input_uncached_usd":0.002615,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7210861248895526}
{"item_id":"semeval2013.d010.s007.t000","gold_sense_keys":["experience%1:09:01::"],"candidates":[{"index":1,"sense_key":"experience%1:11:00::","synset_id":"experience.n.03"},{"index":2,"sense_key":"experience%1:09:01::","synset_id":"experience.n.01"},{"index":3,"sense_key":"experience%1:09:00::","synset_id":"experience.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"experience%1:09:01::","call_ids":["semeval2013.d010.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"experience%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.00521,"input_uncached_usd":0.0024200000000000003,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7362861670553684}
{"item_id":"semeval2013.d010.s008.t000","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":2,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":3,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":4,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d010.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.004675,"input_uncached_usd":0.0028450000000000003,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7497106249211356}
{"item_id":"semeval2013.d010.s008.t001","gold_sense_keys":["experience%1:09:01::"],"candidates":[{"index":1,"sense_key":"experience%1:09:00::","synset_id":"experience.n.02"},{"index":2,"sense_key":"experience%1:09:01::","synset_id":"experience.n.01"},{"index":3,"sense_key":"experience%1:11:00::","synset_id":"experience.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"experience%1:09:01::","call_ids":["semeval2013.d010.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"experience%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.004425,"input_uncached_usd":0.002595,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5066859580110759}
{"item_id":"semeval2013.d010.s008.t002","gold_sense_keys":["area%1:15:01::"],"candidates":[{"index":1,"sense_key":"area%1:26:00::","synset_id":"sphere.n.01"},{"index":2,"sense_key":"area%1:07:00::","synset_id":"area.n.06"},{"index":3,"sense_key":"area%1:08:00::","synset_id":"area.n.03"},{"index":4,"sense_key":"area%1:15:01::","synset_id":"area.n.01"},{"index":5,"sense_key":"area%1:09:00::","synset_id":"area.n.02"},{"index":6,"sense_key":"area%1:06:00::","synset_id":"area.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"area%1:15:01::","call_ids":["semeval2013.d010.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"area%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":678,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01923,"input_uncached_usd":0.0033900000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.144885999965481}
{"item_id":"semeval2013.d010.s008.t003","gold_sense_keys":["level%1:26:01::"],"candidates":[{"index":1,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":2,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":3,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":4,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"},{"index":5,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":6,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":7,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":8,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"level%1:26:01::","call_ids":["semeval2013.d010.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"level%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":219,"reasoning_output_tokens":203},"cost":{"currency":"USD","total_usd":0.010375,"input_uncached_usd":0.0038050000000000002,"input_cached_usd":0.0,"output_usd":0.00657,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.943166041979566}
{"item_id":"semeval2013.d010.s009.t000","gold_sense_keys":["peak%1:26:00::"],"candidates":[{"index":1,"sense_key":"peak%1:23:00::","synset_id":"extremum.n.02"},{"index":2,"sense_key":"peak%1:28:00::","synset_id":"flower.n.03"},{"index":3,"sense_key":"peak%1:25:01::","synset_id":"point.n.08"},{"index":4,"sense_key":"peak%1:06:00::","synset_id":"bill.n.09"},{"index":5,"sense_key":"peak%1:15:01::","synset_id":"vertex.n.02"},{"index":6,"sense_key":"peak%1:15:00::","synset_id":"peak.n.04"},{"index":7,"sense_key":"peak%1:26:00::","synset_id":"acme.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"peak%1:23:00::","call_ids":["semeval2013.d010.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"peak%1:23:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":732,"cached_input_tokens":0,"output_tokens":4646,"reasoning_output_tokens":4582},"cost":{"currency":"USD","total_usd":0.14304,"input_uncached_usd":0.0036600000000000005,"input_cached_usd":0.0,"output_usd":0.13938,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":69.37512791703921}
{"item_id":"semeval2013.d010.s009.t002","gold_sense_keys":["goalkeeper%1:18:00::"],"candidates":[{"index":1,"sense_key":"goalkeeper%1:18:00::","synset_id":"goalkeeper.n.01"},{"index":2,"sense_key":"goalkeeper%1:04:00::","synset_id":"goalkeeper.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"goalkeeper%1:18:00::","call_ids":["semeval2013.d010.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"goalkeeper%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.004635,"input_uncached_usd":0.002445,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.79412095900625}
{"item_id":"semeval2013.d010.s009.t005","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":2,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":3,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":4,"sense_key":"game%1:04:04::","synset_id":"game.n.11"},{"index":5,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":6,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":7,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":8,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":9,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":10,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":11,"sense_key":"game%1:04:02::","synset_id":"game.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.006265,"input_uncached_usd":0.004045,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.206870791967958}
{"item_id":"semeval2013.d010.s009.t006","gold_sense_keys":["london%1:15:00::"],"candidates":[{"index":1,"sense_key":"london%1:15:00::","synset_id":"london.n.01"},{"index":2,"sense_key":"london%1:18:00::","synset_id":"london.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"london%1:15:00::","call_ids":["semeval2013.d010.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"london%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.004235,"input_uncached_usd":0.002435,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1956767920637503}
{"item_id":"semeval2013.d010.s009.t007","gold_sense_keys":["record%1:04:00::"],"candidates":[{"index":1,"sense_key":"record%1:04:01::","synset_id":"record.n.04"},{"index":2,"sense_key":"record%1:23:00::","synset_id":"record.n.03"},{"index":3,"sense_key":"record%1:10:00::","synset_id":"criminal_record.n.01"},{"index":4,"sense_key":"record%1:06:00::","synset_id":"phonograph_record.n.01"},{"index":5,"sense_key":"record%1:21:00::","synset_id":"record.n.07"},{"index":6,"sense_key":"record%1:10:02::","synset_id":"record.n.05"},{"index":7,"sense_key":"record%1:04:00::","synset_id":"record.n.06"},{"index":8,"sense_key":"record%1:10:03::","synset_id":"record.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"record%1:04:00::","call_ids":["semeval2013.d010.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"record%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":802,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":88},"cost":{"currency":"USD","total_usd":0.007130000000000001,"input_uncached_usd":0.0040100000000000005,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7882668330566958}
{"item_id":"semeval2013.d010.s009.t008","gold_sense_keys":["goalkeeper%1:18:00::"],"candidates":[{"index":1,"sense_key":"goalkeeper%1:04:00::","synset_id":"goalkeeper.n.02"},{"index":2,"sense_key":"goalkeeper%1:18:00::","synset_id":"goalkeeper.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"goalkeeper%1:18:00::","call_ids":["semeval2013.d010.s009.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"goalkeeper%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.004965000000000001,"input_uncached_usd":0.002445,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9740872089751065}
{"item_id":"semeval2013.d010.s009.t009","gold_sense_keys":["match%1:11:00::"],"candidates":[{"index":1,"sense_key":"match%1:23:00::","synset_id":"match.n.05"},{"index":2,"sense_key":"match%1:09:00::","synset_id":"match.n.09"},{"index":3,"sense_key":"match%1:06:02::","synset_id":"match.n.03"},{"index":4,"sense_key":"match%1:06:01::","synset_id":"match.n.04"},{"index":5,"sense_key":"match%1:06:00::","synset_id":"match.n.01"},{"index":6,"sense_key":"match%1:14:00::","synset_id":"couple.n.02"},{"index":7,"sense_key":"match%1:18:00::","synset_id":"peer.n.01"},{"index":8,"sense_key":"match%1:11:00::","synset_id":"match.n.02"},{"index":9,"sense_key":"match%1:18:01::","synset_id":"catch.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"match%1:11:00::","call_ids":["semeval2013.d010.s009.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"match%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":717,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.005235,"input_uncached_usd":0.003585,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.633854000014253}
{"item_id":"semeval2013.d010.s010.t000","gold_sense_keys":["today%1:28:01::"],"candidates":[{"index":1,"sense_key":"today%1:28:01::","synset_id":"today.n.02"},{"index":2,"sense_key":"today%1:28:00::","synset_id":"today.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"today%1:28:01::","call_ids":["semeval2013.d010.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"today%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":430,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.00422,"input_uncached_usd":0.00215,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8706387499114498}
{"item_id":"semeval2013.d010.s010.t001","gold_sense_keys":["record%1:04:00::"],"candidates":[{"index":1,"sense_key":"record%1:10:03::","synset_id":"record.n.01"},{"index":2,"sense_key":"record%1:23:00::","synset_id":"record.n.03"},{"index":3,"sense_key":"record%1:04:00::","synset_id":"record.n.06"},{"index":4,"sense_key":"record%1:10:00::","synset_id":"criminal_record.n.01"},{"index":5,"sense_key":"record%1:04:01::","synset_id":"record.n.04"},{"index":6,"sense_key":"record%1:21:00::","synset_id":"record.n.07"},{"index":7,"sense_key":"record%1:10:02::","synset_id":"record.n.05"},{"index":8,"sense_key":"record%1:06:00::","synset_id":"phonograph_record.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"record%1:04:00::","call_ids":["semeval2013.d010.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"record%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":768,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.0063,"input_uncached_usd":0.0038400000000000005,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8684893329627812}
{"item_id":"semeval2013.d010.s010.t002","gold_sense_keys":["match%1:11:00::"],"candidates":[{"index":1,"sense_key":"match%1:06:02::","synset_id":"match.n.03"},{"index":2,"sense_key":"match%1:18:01::","synset_id":"catch.n.03"},{"index":3,"sense_key":"match%1:18:00::","synset_id":"peer.n.01"},{"index":4,"sense_key":"match%1:11:00::","synset_id":"match.n.02"},{"index":5,"sense_key":"match%1:06:01::","synset_id":"match.n.04"},{"index":6,"sense_key":"match%1:23:00::","synset_id":"match.n.05"},{"index":7,"sense_key":"match%1:09:00::","synset_id":"match.n.09"},{"index":8,"sense_key":"match%1:14:00::","synset_id":"couple.n.02"},{"index":9,"sense_key":"match%1:06:00::","synset_id":"match.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"match%1:11:00::","call_ids":["semeval2013.d010.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"match%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":684,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.0048000000000000004,"input_uncached_usd":0.0034200000000000003,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4268251250032336}
{"item_id":"semeval2013.d010.s011.t000","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":2,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":3,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":4,"sense_key":"player%1:18:03::","synset_id":"player.n.05"},{"index":5,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d010.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.0042450000000000005,"input_uncached_usd":0.0027150000000000004,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3954337500035763}
{"item_id":"semeval2013.d010.s011.t001","gold_sense_keys":["match%1:11:00::"],"candidates":[{"index":1,"sense_key":"match%1:09:00::","synset_id":"match.n.09"},{"index":2,"sense_key":"match%1:23:00::","synset_id":"match.n.05"},{"index":3,"sense_key":"match%1:14:00::","synset_id":"couple.n.02"},{"index":4,"sense_key":"match%1:18:00::","synset_id":"peer.n.01"},{"index":5,"sense_key":"match%1:06:02::","synset_id":"match.n.03"},{"index":6,"sense_key":"match%1:06:01::","synset_id":"match.n.04"},{"index":7,"sense_key":"match%1:18:01::","synset_id":"catch.n.03"},{"index":8,"sense_key":"match%1:06:00::","synset_id":"match.n.01"},{"index":9,"sense_key":"match%1:11:00::","synset_id":"match.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"match%1:11:00::","call_ids":["semeval2013.d010.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"match%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":703,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.005255,"input_uncached_usd":0.0035150000000000003,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8943345419829711}
{"item_id":"semeval2013.d010.s012.t000","gold_sense_keys":["week%1:28:02::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"week%1:28:02::","call_ids":["semeval2013.d010.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"week%1:28:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":227,"reasoning_output_tokens":211},"cost":{"currency":"USD","total_usd":0.009075000000000001,"input_uncached_usd":0.002265,"input_cached_usd":0.0,"output_usd":0.00681,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.2335458339657634}
{"item_id":"semeval2013.d010.s012.t001","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":2,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":3,"sense_key":"player%1:18:03::","synset_id":"player.n.05"},{"index":4,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":5,"sense_key":"player%1:18:01::","synset_id":"player.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d010.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.004495,"input_uncached_usd":0.002575,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3163310419768095}
{"item_id":"semeval2013.d010.s012.t002","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d010.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":399,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.003585,"input_uncached_usd":0.0019950000000000002,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.173773083020933}
{"item_id":"semeval2013.d010.s012.t003","gold_sense_keys":["match%1:11:00::"],"candidates":[{"index":1,"sense_key":"match%1:09:00::","synset_id":"match.n.09"},{"index":2,"sense_key":"match%1:23:00::","synset_id":"match.n.05"},{"index":3,"sense_key":"match%1:14:00::","synset_id":"couple.n.02"},{"index":4,"sense_key":"match%1:06:01::","synset_id":"match.n.04"},{"index":5,"sense_key":"match%1:18:01::","synset_id":"catch.n.03"},{"index":6,"sense_key":"match%1:06:02::","synset_id":"match.n.03"},{"index":7,"sense_key":"match%1:11:00::","synset_id":"match.n.02"},{"index":8,"sense_key":"match%1:06:00::","synset_id":"match.n.01"},{"index":9,"sense_key":"match%1:18:00::","synset_id":"peer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"match%1:11:00::","call_ids":["semeval2013.d010.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"match%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":676,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.0049700000000000005,"input_uncached_usd":0.00338,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.490661625051871}
{"item_id":"semeval2013.d010.s012.t004","gold_sense_keys":["striker%1:18:02::"],"candidates":[{"index":1,"sense_key":"striker%1:18:00::","synset_id":"striker.n.03"},{"index":2,"sense_key":"striker%1:18:02::","synset_id":"striker.n.01"},{"index":3,"sense_key":"striker%1:18:03::","synset_id":"hitter.n.02"},{"index":4,"sense_key":"striker%1:18:01::","synset_id":"striker.n.02"},{"index":5,"sense_key":"striker%1:06:00::","synset_id":"striker.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"striker%1:18:02::","call_ids":["semeval2013.d010.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"striker%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":42,"reasoning_output_tokens":27},"cost":{"currency":"USD","total_usd":0.0036550000000000003,"input_uncached_usd":0.002395,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7501439579064026}
{"item_id":"semeval2013.d010.s013.t000","gold_sense_keys":["number%1:07:00::"],"candidates":[{"index":1,"sense_key":"number%1:10:05::","synset_id":"number.n.10"},{"index":2,"sense_key":"number%1:23:00::","synset_id":"number.n.02"},{"index":3,"sense_key":"number%1:07:01::","synset_id":"number.n.09"},{"index":4,"sense_key":"number%1:14:00::","synset_id":"number.n.07"},{"index":5,"sense_key":"number%1:07:00::","synset_id":"number.n.01"},{"index":6,"sense_key":"number%1:10:03::","synset_id":"phone_number.n.01"},{"index":7,"sense_key":"number%1:10:01::","synset_id":"numeral.n.01"},{"index":8,"sense_key":"number%1:10:04::","synset_id":"act.n.04"},{"index":9,"sense_key":"number%1:10:02::","synset_id":"number.n.08"},{"index":10,"sense_key":"number%1:06:00::","synset_id":"number.n.11"},{"index":11,"sense_key":"number%1:10:00::","synset_id":"issue.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"number%1:23:00::","call_ids":["semeval2013.d010.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"number%1:23:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":864,"cached_input_tokens":0,"output_tokens":241,"reasoning_output_tokens":225},"cost":{"currency":"USD","total_usd":0.011550000000000001,"input_uncached_usd":0.00432,"input_cached_usd":0.0,"output_usd":0.00723,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.62402520806063}
{"item_id":"semeval2013.d010.s013.t001","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":2,"sense_key":"game%1:04:04::","synset_id":"game.n.11"},{"index":3,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":8,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":9,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":10,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":11,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":802,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.00638,"input_uncached_usd":0.0040100000000000005,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1083535000216216}
{"item_id":"semeval2013.d010.s015.t000","gold_sense_keys":["side%1:14:00::"],"candidates":[{"index":1,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":2,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":3,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":4,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":5,"sense_key":"side%1:11:00::","synset_id":"english.n.04"},{"index":6,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":7,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":8,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":9,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":10,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":11,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":12,"sense_key":"side%1:08:00::","synset_id":"side.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"side%1:14:00::","call_ids":["semeval2013.d010.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"side%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":886,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.007070000000000001,"input_uncached_usd":0.004430000000000001,"input_cached_usd":0.0,"output_usd":0.00264,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1835912499809638}
{"item_id":"semeval2013.d010.s015.t001","gold_sense_keys":["match%1:11:00::"],"candidates":[{"index":1,"sense_key":"match%1:11:00::","synset_id":"match.n.02"},{"index":2,"sense_key":"match%1:14:00::","synset_id":"couple.n.02"},{"index":3,"sense_key":"match%1:06:00::","synset_id":"match.n.01"},{"index":4,"sense_key":"match%1:18:00::","synset_id":"peer.n.01"},{"index":5,"sense_key":"match%1:23:00::","synset_id":"match.n.05"},{"index":6,"sense_key":"match%1:06:01::","synset_id":"match.n.04"},{"index":7,"sense_key":"match%1:06:02::","synset_id":"match.n.03"},{"index":8,"sense_key":"match%1:18:01::","synset_id":"catch.n.03"},{"index":9,"sense_key":"match%1:09:00::","synset_id":"match.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"match%1:11:00::","call_ids":["semeval2013.d010.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"match%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.00574,"input_uncached_usd":0.00355,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0686817499808967}
{"item_id":"semeval2013.d010.s015.t002","gold_sense_keys":["striker%1:18:02::"],"candidates":[{"index":1,"sense_key":"striker%1:18:01::","synset_id":"striker.n.02"},{"index":2,"sense_key":"striker%1:18:02::","synset_id":"striker.n.01"},{"index":3,"sense_key":"striker%1:06:00::","synset_id":"striker.n.05"},{"index":4,"sense_key":"striker%1:18:00::","synset_id":"striker.n.03"},{"index":5,"sense_key":"striker%1:18:03::","synset_id":"hitter.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"striker%1:18:02::","call_ids":["semeval2013.d010.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"striker%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.004095,"input_uncached_usd":0.002565,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":71.56100470898673}
{"item_id":"semeval2013.d010.s016.t000","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":2,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":3,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":4,"sense_key":"game%1:04:04::","synset_id":"game.n.11"},{"index":5,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":6,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":7,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":8,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":9,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":10,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":11,"sense_key":"game%1:04:03::","synset_id":"game.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":781,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.006845,"input_uncached_usd":0.0039050000000000005,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8603197919437662}
{"item_id":"semeval2013.d010.s018.t000","gold_sense_keys":["match%1:11:00::"],"candidates":[{"index":1,"sense_key":"match%1:06:00::","synset_id":"match.n.01"},{"index":2,"sense_key":"match%1:18:01::","synset_id":"catch.n.03"},{"index":3,"sense_key":"match%1:23:00::","synset_id":"match.n.05"},{"index":4,"sense_key":"match%1:14:00::","synset_id":"couple.n.02"},{"index":5,"sense_key":"match%1:11:00::","synset_id":"match.n.02"},{"index":6,"sense_key":"match%1:06:01::","synset_id":"match.n.04"},{"index":7,"sense_key":"match%1:06:02::","synset_id":"match.n.03"},{"index":8,"sense_key":"match%1:09:00::","synset_id":"match.n.09"},{"index":9,"sense_key":"match%1:18:00::","synset_id":"peer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"match%1:11:00::","call_ids":["semeval2013.d010.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"match%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":722,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":81},"cost":{"currency":"USD","total_usd":0.00652,"input_uncached_usd":0.0036100000000000004,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.42864879197441}
{"item_id":"semeval2013.d010.s019.t000","gold_sense_keys":["side%1:14:00::"],"candidates":[{"index":1,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":2,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":3,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":4,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":5,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":6,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":7,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":8,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":9,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":10,"sense_key":"side%1:11:00::","synset_id":"english.n.04"},{"index":11,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":12,"sense_key":"side%1:15:02::","synset_id":"side.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"side%1:14:00::","call_ids":["semeval2013.d010.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"side%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":889,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":113},"cost":{"currency":"USD","total_usd":0.008315,"input_uncached_usd":0.004445,"input_cached_usd":0.0,"output_usd":0.00387,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.453363790991716}
{"item_id":"semeval2013.d010.s019.t001","gold_sense_keys":["presence%1:04:00::"],"candidates":[{"index":1,"sense_key":"presence%1:26:00::","synset_id":"presence.n.01"},{"index":2,"sense_key":"presence%1:07:00::","synset_id":"bearing.n.03"},{"index":3,"sense_key":"presence%1:18:00::","synset_id":"presence.n.03"},{"index":4,"sense_key":"presence%1:04:00::","synset_id":"presence.n.06"},{"index":5,"sense_key":"presence%1:15:00::","synset_id":"presence.n.02"},{"index":6,"sense_key":"presence%1:09:00::","synset_id":"presence.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"presence%1:04:00::","call_ids":["semeval2013.d010.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"presence%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":123},"cost":{"currency":"USD","total_usd":0.0070350000000000005,"input_uncached_usd":0.0028650000000000004,"input_cached_usd":0.0,"output_usd":0.00417,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5408715420635417}
{"item_id":"semeval2013.d010.s020.t000","gold_sense_keys":["rest%1:24:00::"],"candidates":[{"index":1,"sense_key":"rest%1:04:00::","synset_id":"rest.n.02"},{"index":2,"sense_key":"rest%1:24:00::","synset_id":"remainder.n.01"},{"index":3,"sense_key":"rest%1:10:00::","synset_id":"rest.n.07"},{"index":4,"sense_key":"rest%1:06:00::","synset_id":"rest.n.06"},{"index":5,"sense_key":"rest%1:28:00::","synset_id":"respite.n.04"},{"index":6,"sense_key":"rest%1:26:00::","synset_id":"rest.n.04"},{"index":7,"sense_key":"rest%1:26:01::","synset_id":"rest.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"rest%1:24:00::","call_ids":["semeval2013.d010.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"rest%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":673,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.005795,"input_uncached_usd":0.0033650000000000004,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1527480840450153}
{"item_id":"semeval2013.d010.s021.t000","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":2,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":3,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":4,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":5,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":6,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":9,"sense_key":"game%1:04:04::","synset_id":"game.n.11"},{"index":10,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":11,"sense_key":"game%1:04:00::","synset_id":"game.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":776,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.00643,"input_uncached_usd":0.00388,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3130551249487326}
{"item_id":"semeval2013.d010.s022.t000","gold_sense_keys":["scorer%1:18:00::"],"candidates":[{"index":1,"sense_key":"scorer%1:18:00::","synset_id":"scorer.n.01"},{"index":2,"sense_key":"scorer%1:18:01::","synset_id":"scorer.n.02"},{"index":3,"sense_key":"scorer%1:18:02::","synset_id":"scorekeeper.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"scorer%1:18:00::","call_ids":["semeval2013.d010.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"scorer%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0039900000000000005,"input_uncached_usd":0.0021000000000000003,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9337210829835385}
{"item_id":"semeval2013.d010.s022.t001","gold_sense_keys":["goal%1:04:00::"],"candidates":[{"index":1,"sense_key":"goal%1:04:00::","synset_id":"goal.n.04"},{"index":2,"sense_key":"goal%1:15:00::","synset_id":"finish.n.04"},{"index":3,"sense_key":"goal%1:06:00::","synset_id":"goal.n.03"},{"index":4,"sense_key":"goal%1:09:00::","synset_id":"goal.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"goal%1:04:00::","call_ids":["semeval2013.d010.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"goal%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.004030000000000001,"input_uncached_usd":0.0025900000000000003,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5261939580086619}
{"item_id":"semeval2013.d010.s023.t000","gold_sense_keys":["point%1:23:01::"],"candidates":[{"index":1,"sense_key":"point%1:06:01::","synset_id":"point.n.25"},{"index":2,"sense_key":"point%1:10:00::","synset_id":"period.n.07"},{"index":3,"sense_key":"point%1:06:00::","synset_id":"point.n.15"},{"index":4,"sense_key":"point%1:06:02::","synset_id":"point.n.24"},{"index":5,"sense_key":"point%1:15:00::","synset_id":"point.n.02"},{"index":6,"sense_key":"point%1:10:03::","synset_id":"item.n.01"},{"index":7,"sense_key":"point%1:24:00::","synset_id":"compass_point.n.01"},{"index":8,"sense_key":"point%1:10:01::","synset_id":"point.n.03"},{"index":9,"sense_key":"point%1:28:00::","synset_id":"point.n.06"},{"index":10,"sense_key":"point%1:25:00::","synset_id":"point.n.08"},{"index":11,"sense_key":"point%1:07:02::","synset_id":"point.n.23"},{"index":12,"sense_key":"point%1:17:02::","synset_id":"point.n.11"},{"index":13,"sense_key":"point%1:06:03::","synset_id":"distributor_point.n.01"},{"index":14,"sense_key":"point%1:07:00::","synset_id":"point.n.22"},{"index":15,"sense_key":"point%1:23:01::","synset_id":"point.n.10"},{"index":16,"sense_key":"point%1:21:00::","synset_id":"point.n.18"},{"index":17,"sense_key":"point%1:10:04::","synset_id":"decimal_point.n.01"},{"index":18,"sense_key":"point%1:23:00::","synset_id":"point.n.17"},{"index":19,"sense_key":"point%1:26:00::","synset_id":"degree.n.02"},{"index":20,"sense_key":"point%1:07:01::","synset_id":"point.n.14"},{"index":21,"sense_key":"point%1:10:02::","synset_id":"point.n.20"},{"index":22,"sense_key":"point%1:24:04::","synset_id":"point.n.13"},{"index":23,"sense_key":"point%1:25:02::","synset_id":"point.n.09"},{"index":24,"sense_key":"point%1:09:02::","synset_id":"point.n.07"},{"index":25,"sense_key":"point%1:09:01::","synset_id":"detail.n.01"},{"index":26,"sense_key":"point%1:09:00::","synset_id":"point.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"point%1:23:01::","call_ids":["semeval2013.d010.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"point%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1390,"cached_input_tokens":0,"output_tokens":241,"reasoning_output_tokens":226},"cost":{"currency":"USD","total_usd":0.014180000000000002,"input_uncached_usd":0.0069500000000000004,"input_cached_usd":0.0,"output_usd":0.00723,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.691108500002883}
{"item_id":"semeval2013.d011.s000.t001","gold_sense_keys":["burden%1:09:01::"],"candidates":[{"index":1,"sense_key":"burden%1:09:01::","synset_id":"burden.n.01"},{"index":2,"sense_key":"burden%1:10:00::","synset_id":"effect.n.04"},{"index":3,"sense_key":"burden%1:09:00::","synset_id":"burden.n.04"},{"index":4,"sense_key":"burden%1:06:01::","synset_id":"load.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"burden%1:09:01::","call_ids":["semeval2013.d011.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"burden%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":359,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.003925,"input_uncached_usd":0.0017950000000000002,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1701426659710705}
{"item_id":"semeval2013.d011.s000.t003","gold_sense_keys":["economy%1:14:00::"],"candidates":[{"index":1,"sense_key":"economy%1:14:00::","synset_id":"economy.n.01"},{"index":2,"sense_key":"economy%1:09:01::","synset_id":"economy.n.02"},{"index":3,"sense_key":"economy%1:04:00::","synset_id":"economy.n.04"},{"index":4,"sense_key":"economy%1:07:00::","synset_id":"economy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"economy%1:14:00::","call_ids":["semeval2013.d011.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"economy%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":366,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.00345,"input_uncached_usd":0.0018300000000000002,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4885394999291748}
{"item_id":"semeval2013.d011.s001.t000","gold_sense_keys":["immigration%1:04:00::"],"candidates":[{"index":1,"sense_key":"immigration%1:14:00::","synset_id":"immigration.n.02"},{"index":2,"sense_key":"immigration%1:04:00::","synset_id":"immigration.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"immigration%1:04:00::","call_ids":["semeval2013.d011.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"immigration%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":336,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.0036,"input_uncached_usd":0.00168,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5604935829760507}
{"item_id":"semeval2013.d011.s001.t001","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"},{"index":2,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":3,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"policy%1:09:00::","call_ids":["semeval2013.d011.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":257,"reasoning_output_tokens":241},"cost":{"currency":"USD","total_usd":0.009625000000000002,"input_uncached_usd":0.0019150000000000003,"input_cached_usd":0.0,"output_usd":0.00771,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.112881457898766}
{"item_id":"semeval2013.d011.s001.t002","gold_sense_keys":["aspect%1:09:00::"],"candidates":[{"index":1,"sense_key":"aspect%1:24:00::","synset_id":"aspect.n.04"},{"index":2,"sense_key":"aspect%1:09:01::","synset_id":"view.n.02"},{"index":3,"sense_key":"aspect%1:07:02::","synset_id":"aspect.n.02"},{"index":4,"sense_key":"aspect%1:09:00::","synset_id":"aspect.n.01"},{"index":5,"sense_key":"aspect%1:07:01::","synset_id":"expression.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"aspect%1:09:00::","call_ids":["semeval2013.d011.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"aspect%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018025,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.579081999952905}
{"item_id":"semeval2013.d011.s001.t003","gold_sense_keys":["congestion%1:26:01::"],"candidates":[{"index":1,"sense_key":"congestion%1:26:00::","synset_id":"congestion.n.01"},{"index":2,"sense_key":"congestion%1:26:01::","synset_id":"congestion.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"congestion%1:26:01::","call_ids":["semeval2013.d011.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"congestion%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":320,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.00391,"input_uncached_usd":0.0016,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7679269999498501}
{"item_id":"semeval2013.d011.s001.t006","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"},{"index":2,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":3,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"policy%1:09:00::","call_ids":["semeval2013.d011.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":136},"cost":{"currency":"USD","total_usd":0.006475000000000001,"input_uncached_usd":0.0019150000000000003,"input_cached_usd":0.0,"output_usd":0.00456,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1330205419799313}
{"item_id":"semeval2013.d011.s001.t007","gold_sense_keys":["number%1:07:00::"],"candidates":[{"index":1,"sense_key":"number%1:10:03::","synset_id":"phone_number.n.01"},{"index":2,"sense_key":"number%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"number%1:07:00::","synset_id":"number.n.01"},{"index":4,"sense_key":"number%1:14:00::","synset_id":"number.n.07"},{"index":5,"sense_key":"number%1:23:00::","synset_id":"number.n.02"},{"index":6,"sense_key":"number%1:06:00::","synset_id":"number.n.11"},{"index":7,"sense_key":"number%1:10:04::","synset_id":"act.n.04"},{"index":8,"sense_key":"number%1:07:01::","synset_id":"number.n.09"},{"index":9,"sense_key":"number%1:10:01::","synset_id":"numeral.n.01"},{"index":10,"sense_key":"number%1:10:02::","synset_id":"number.n.08"},{"index":11,"sense_key":"number%1:10:05::","synset_id":"number.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"number%1:07:00::","call_ids":["semeval2013.d011.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"number%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.00678,"input_uncached_usd":0.00369,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5700152090284973}
{"item_id":"semeval2013.d011.s001.t008","gold_sense_keys":["compatibility%1:07:00::"],"candidates":[{"index":1,"sense_key":"compatibility%1:07:00::","synset_id":"compatibility.n.02"},{"index":2,"sense_key":"compatibility%1:12:00::","synset_id":"compatibility.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compatibility%1:07:00::","call_ids":["semeval2013.d011.s001.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compatibility%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":306,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.00312,"input_uncached_usd":0.0015300000000000001,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5908394160214812}
{"item_id":"semeval2013.d011.s001.t009","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":2,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"},{"index":3,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"policy%1:09:00::","call_ids":["semeval2013.d011.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":276,"reasoning_output_tokens":261},"cost":{"currency":"USD","total_usd":0.010195000000000001,"input_uncached_usd":0.0019150000000000003,"input_cached_usd":0.0,"output_usd":0.008280000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.350376290967688}
{"item_id":"semeval2013.d011.s001.t010","gold_sense_keys":["status%1:26:00::"],"candidates":[{"index":1,"sense_key":"status%1:26:00::","synset_id":"status.n.01"},{"index":2,"sense_key":"status%1:26:01::","synset_id":"condition.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"status%1:26:00::","call_ids":["semeval2013.d011.s001.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"status%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":349,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":133},"cost":{"currency":"USD","total_usd":0.006215000000000001,"input_uncached_usd":0.0017450000000000002,"input_cached_usd":0.0,"output_usd":0.00447,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6972570839570835}
{"item_id":"semeval2013.d011.s001.t011","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":2,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":3,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":4,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":5,"sense_key":"country%1:15:01::","synset_id":"country.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d011.s001.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":286,"reasoning_output_tokens":270},"cost":{"currency":"USD","total_usd":0.01109,"input_uncached_usd":0.00251,"input_cached_usd":0.0,"output_usd":0.00858,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.832491124980152}
{"item_id":"semeval2013.d011.s001.t012","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":2,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":3,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":4,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":5,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d011.s001.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018349999999999998,"input_uncached_usd":0.00251,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.071820541052148}
{"item_id":"semeval2013.d011.s002.t000","gold_sense_keys":["harassment%1:04:00::"],"candidates":[{"index":1,"sense_key":"harassment%1:04:00::","synset_id":"harassment.n.02"},{"index":2,"sense_key":"harassment%1:12:00::","synset_id":"harassment.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"harassment%1:04:00::","call_ids":["semeval2013.d011.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"harassment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":347,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.0037750000000000006,"input_uncached_usd":0.0017350000000000002,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4574096660362557}
{"item_id":"semeval2013.d011.s002.t001","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"},{"index":2,"sense_key":"student%1:18:00::","synset_id":"student.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"student%1:18:00::","call_ids":["semeval2013.d011.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":344,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.0032500000000000003,"input_uncached_usd":0.0017200000000000002,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.431448542047292}
{"item_id":"semeval2013.d011.s002.t002","gold_sense_keys":["france%1:15:00::"],"candidates":[{"index":1,"sense_key":"france%1:18:00::","synset_id":"france.n.02"},{"index":2,"sense_key":"france%1:15:00::","synset_id":"france.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"france%1:15:00::","call_ids":["semeval2013.d011.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"france%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":347,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.0031450000000000002,"input_uncached_usd":0.0017350000000000002,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9411437079543248}
{"item_id":"semeval2013.d011.s002.t003","gold_sense_keys":["talent%1:18:00::"],"candidates":[{"index":1,"sense_key":"talent%1:09:00::","synset_id":"endowment.n.01"},{"index":2,"sense_key":"talent%1:18:00::","synset_id":"talent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"talent%1:18:00::","call_ids":["semeval2013.d011.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"talent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":325,"cached_input_tokens":0,"output_tokens":225,"reasoning_output_tokens":209},"cost":{"currency":"USD","total_usd":0.008375,"input_uncached_usd":0.0016250000000000001,"input_cached_usd":0.0,"output_usd":0.00675,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7350011250237003}
{"item_id":"semeval2013.d011.s002.t005","gold_sense_keys":["waiter%1:18:00::"],"candidates":[{"index":1,"sense_key":"waiter%1:18:00::","synset_id":"waiter.n.01"},{"index":2,"sense_key":"waiter%1:18:01::","synset_id":"waiter.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"waiter%1:18:00::","call_ids":["semeval2013.d011.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"waiter%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":324,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.0032700000000000003,"input_uncached_usd":0.0016200000000000001,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.434466667007655}
{"item_id":"semeval2013.d011.s003.t000","gold_sense_keys":["viewpoint%1:09:00::"],"candidates":[{"index":1,"sense_key":"viewpoint%1:09:00::","synset_id":"point_of_view.n.01"},{"index":2,"sense_key":"viewpoint%1:15:00::","synset_id":"vantage_point.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"viewpoint%1:09:00::","call_ids":["semeval2013.d011.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"viewpoint%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.0033550000000000003,"input_uncached_usd":0.0019150000000000003,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3339212080463767}
{"item_id":"semeval2013.d011.s004.t000","gold_sense_keys":["lot%1:23:00::"],"candidates":[{"index":1,"sense_key":"lot%1:15:00::","synset_id":"lot.n.02"},{"index":2,"sense_key":"lot%1:23:00::","synset_id":"batch.n.02"},{"index":3,"sense_key":"lot%1:18:00::","synset_id":"lot.n.07"},{"index":4,"sense_key":"lot%1:26:00::","synset_id":"fortune.n.04"},{"index":5,"sense_key":"lot%1:06:00::","synset_id":"draw.n.04"},{"index":6,"sense_key":"lot%1:14:00::","synset_id":"set.n.05"},{"index":7,"sense_key":"lot%1:14:01::","synset_id":"bunch.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"lot%1:23:00::","call_ids":["semeval2013.d011.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"lot%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":679,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.005975,"input_uncached_usd":0.0033950000000000004,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.062384874909185}
{"item_id":"semeval2013.d011.s004.t002","gold_sense_keys":["left%1:14:00::"],"candidates":[{"index":1,"sense_key":"left%1:06:00::","synset_id":"left_field.n.01"},{"index":2,"sense_key":"left%1:08:00::","synset_id":"left.n.03"},{"index":3,"sense_key":"left%1:04:00::","synset_id":"left.n.05"},{"index":4,"sense_key":"left%1:14:00::","synset_id":"left.n.02"},{"index":5,"sense_key":"left%1:15:00::","synset_id":"left.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"left%1:14:00::","call_ids":["semeval2013.d011.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"left%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004855,"input_uncached_usd":0.0028150000000000002,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9026832500239834}
{"item_id":"semeval2013.d011.s004.t003","gold_sense_keys":["right%1:14:00::"],"candidates":[{"index":1,"sense_key":"right%1:08:00::","synset_id":"right.n.05"},{"index":2,"sense_key":"right%1:06:00::","synset_id":"right_field.n.01"},{"index":3,"sense_key":"right%1:07:01::","synset_id":"right.n.07"},{"index":4,"sense_key":"right%1:21:00::","synset_id":"right.n.08"},{"index":5,"sense_key":"right%1:07:00::","synset_id":"right.n.01"},{"index":6,"sense_key":"right%1:15:00::","synset_id":"right.n.02"},{"index":7,"sense_key":"right%1:04:00::","synset_id":"right.n.06"},{"index":8,"sense_key":"right%1:14:00::","synset_id":"right.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"right%1:14:00::","call_ids":["semeval2013.d011.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"right%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.005955,"input_uncached_usd":0.0036750000000000003,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9142424169695005}
{"item_id":"semeval2013.d011.s004.t004","gold_sense_keys":["subject%1:09:01::","subject%1:10:00::"],"candidates":[{"index":1,"sense_key":"subject%1:10:02::","synset_id":"subject.n.08"},{"index":2,"sense_key":"subject%1:10:00::","synset_id":"subject.n.01"},{"index":3,"sense_key":"subject%1:10:01::","synset_id":"subject.n.05"},{"index":4,"sense_key":"subject%1:09:01::","synset_id":"topic.n.02"},{"index":5,"sense_key":"subject%1:18:01::","synset_id":"subject.n.06"},{"index":6,"sense_key":"subject%1:09:00::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"subject%1:18:00::","synset_id":"national.n.01"},{"index":8,"sense_key":"subject%1:06:00::","synset_id":"subject.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"subject%1:10:00::","call_ids":["semeval2013.d011.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"subject%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":334,"reasoning_output_tokens":318},"cost":{"currency":"USD","total_usd":0.013665,"input_uncached_usd":0.0036450000000000002,"input_cached_usd":0.0,"output_usd":0.01002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.891077792039141}
{"item_id":"semeval2013.d011.s005.t000","gold_sense_keys":["right%1:14:00::"],"candidates":[{"index":1,"sense_key":"right%1:08:00::","synset_id":"right.n.05"},{"index":2,"sense_key":"right%1:15:00::","synset_id":"right.n.02"},{"index":3,"sense_key":"right%1:04:00::","synset_id":"right.n.06"},{"index":4,"sense_key":"right%1:06:00::","synset_id":"right_field.n.01"},{"index":5,"sense_key":"right%1:14:00::","synset_id":"right.n.04"},{"index":6,"sense_key":"right%1:07:00::","synset_id":"right.n.01"},{"index":7,"sense_key":"right%1:07:01::","synset_id":"right.n.07"},{"index":8,"sense_key":"right%1:21:00::","synset_id":"right.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"right%1:14:00::","call_ids":["semeval2013.d011.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"right%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.005575,"input_uncached_usd":0.0037150000000000004,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7368942919420078}
{"item_id":"semeval2013.d011.s005.t001","gold_sense_keys":["tone%1:07:03::"],"candidates":[{"index":1,"sense_key":"tone%1:09:00::","synset_id":"tone.n.07"},{"index":2,"sense_key":"tone%1:07:03::","synset_id":"tone.n.10"},{"index":3,"sense_key":"tone%1:26:02::","synset_id":"tonicity.n.01"},{"index":4,"sense_key":"tone%1:10:01::","synset_id":"tone.n.01"},{"index":5,"sense_key":"tone%1:26:01::","synset_id":"spirit.n.02"},{"index":6,"sense_key":"tone%1:07:02::","synset_id":"tone.n.02"},{"index":7,"sense_key":"tone%1:07:01::","synset_id":"timbre.n.01"},{"index":8,"sense_key":"tone%1:07:00::","synset_id":"shade.n.02"},{"index":9,"sense_key":"tone%1:10:00::","synset_id":"tone.n.09"},{"index":10,"sense_key":"tone%1:10:02::","synset_id":"note.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tone%1:07:03::","call_ids":["semeval2013.d011.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tone%1:07:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":885,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020265,"input_uncached_usd":0.004425,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.302853500004858}
{"item_id":"semeval2013.d011.s005.t003","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":2,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"problem%1:26:00::","call_ids":["semeval2013.d011.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"problem%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018215000000000002,"input_uncached_usd":0.0023750000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.290314333047718}
{"item_id":"semeval2013.d011.s005.t004","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":2,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":3,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":4,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"country%1:15:01::","synset_id":"country.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d011.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":603,"cached_input_tokens":0,"output_tokens":2075,"reasoning_output_tokens":2041},"cost":{"currency":"USD","total_usd":0.065265,"input_uncached_usd":0.0030150000000000003,"input_cached_usd":0.0,"output_usd":0.06225,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":31.446827915962785}
{"item_id":"semeval2013.d011.s005.t005","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"},{"index":2,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"},{"index":3,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"},{"index":4,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"worker%1:18:00::","call_ids":["semeval2013.d011.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":191,"reasoning_output_tokens":175},"cost":{"currency":"USD","total_usd":0.00833,"input_uncached_usd":0.0026000000000000003,"input_cached_usd":0.0,"output_usd":0.00573,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7394616659730673}
{"item_id":"semeval2013.d011.s005.t006","gold_sense_keys":["left%1:14:00::"],"candidates":[{"index":1,"sense_key":"left%1:08:00::","synset_id":"left.n.03"},{"index":2,"sense_key":"left%1:14:00::","synset_id":"left.n.02"},{"index":3,"sense_key":"left%1:06:00::","synset_id":"left_field.n.01"},{"index":4,"sense_key":"left%1:04:00::","synset_id":"left.n.05"},{"index":5,"sense_key":"left%1:15:00::","synset_id":"left.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"left%1:14:00::","call_ids":["semeval2013.d011.s005.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"left%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.0048400000000000006,"input_uncached_usd":0.00286,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9130112499697134}
{"item_id":"semeval2013.d011.s005.t007","gold_sense_keys":["france%1:15:00::"],"candidates":[{"index":1,"sense_key":"france%1:15:00::","synset_id":"france.n.01"},{"index":2,"sense_key":"france%1:18:00::","synset_id":"france.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"france%1:15:00::","call_ids":["semeval2013.d011.s005.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"france%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.0038050000000000002,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6398037920007482}
{"item_id":"semeval2013.d011.s005.t009","gold_sense_keys":["misery%1:26:00::"],"candidates":[{"index":1,"sense_key":"misery%1:26:00::","synset_id":"misery.n.01"},{"index":2,"sense_key":"misery%1:12:00::","synset_id":"misery.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"misery%1:26:00::","call_ids":["semeval2013.d011.s005.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"misery%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.005725000000000001,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.598159666056745}
{"item_id":"semeval2013.d011.s006.t000","gold_sense_keys":["difference%1:07:00::"],"candidates":[{"index":1,"sense_key":"difference%1:24:00::","synset_id":"difference.n.04"},{"index":2,"sense_key":"difference%1:23:00::","synset_id":"remainder.n.03"},{"index":3,"sense_key":"difference%1:10:00::","synset_id":"dispute.n.01"},{"index":4,"sense_key":"difference%1:11:00::","synset_id":"deviation.n.01"},{"index":5,"sense_key":"difference%1:07:00::","synset_id":"difference.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"difference%1:07:00::","call_ids":["semeval2013.d011.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"difference%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":581,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":112},"cost":{"currency":"USD","total_usd":0.006745,"input_uncached_usd":0.002905,"input_cached_usd":0.0,"output_usd":0.00384,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.727925290935673}
{"item_id":"semeval2013.d011.s006.t001","gold_sense_keys":["degree%1:07:00::","degree%1:26:01::"],"candidates":[{"index":1,"sense_key":"degree%1:23:00::","synset_id":"degree.n.04"},{"index":2,"sense_key":"degree%1:07:00::","synset_id":"degree.n.01"},{"index":3,"sense_key":"degree%1:10:00::","synset_id":"academic_degree.n.01"},{"index":4,"sense_key":"degree%1:09:00::","synset_id":"degree.n.05"},{"index":5,"sense_key":"degree%1:07:01::","synset_id":"degree.n.07"},{"index":6,"sense_key":"degree%1:23:03::","synset_id":"degree.n.06"},{"index":7,"sense_key":"degree%1:26:01::","synset_id":"degree.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"degree%1:07:00::","call_ids":["semeval2013.d011.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"degree%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.034765,"input_uncached_usd":0.0033850000000000004,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":18.449782749987207}
{"item_id":"semeval2013.d011.s006.t002","gold_sense_keys":["generosity%1:07:00::"],"candidates":[{"index":1,"sense_key":"generosity%1:07:00::","synset_id":"generosity.n.01"},{"index":2,"sense_key":"generosity%1:04:00::","synset_id":"generosity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"generosity%1:07:00::","call_ids":["semeval2013.d011.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"generosity%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":436,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":121},"cost":{"currency":"USD","total_usd":0.0062900000000000005,"input_uncached_usd":0.00218,"input_cached_usd":0.0,"output_usd":0.00411,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3584587919758633}
{"item_id":"semeval2013.d011.s007.t000","gold_sense_keys":["fear%1:12:01::"],"candidates":[{"index":1,"sense_key":"fear%1:12:00::","synset_id":"fear.n.01"},{"index":2,"sense_key":"fear%1:12:01::","synset_id":"concern.n.02"},{"index":3,"sense_key":"fear%1:12:02::","synset_id":"fear.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fear%1:12:00::","call_ids":["semeval2013.d011.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fear%1:12:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":86},"cost":{"currency":"USD","total_usd":0.00544,"input_uncached_usd":0.0024100000000000002,"input_cached_usd":0.0,"output_usd":0.00303,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.215632583014667}
{"item_id":"semeval2013.d011.s007.t001","gold_sense_keys":["invasion%1:11:00::"],"candidates":[{"index":1,"sense_key":"invasion%1:11:01::","synset_id":"invasion.n.03"},{"index":2,"sense_key":"invasion%1:04:00::","synset_id":"invasion.n.01"},{"index":3,"sense_key":"invasion%1:11:00::","synset_id":"invasion.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"invasion%1:11:00::","call_ids":["semeval2013.d011.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"invasion%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.004675,"input_uncached_usd":0.0023350000000000003,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0663490829756483}
{"item_id":"semeval2013.d011.s007.t002","gold_sense_keys":["territory%1:15:01::"],"candidates":[{"index":1,"sense_key":"territory%1:15:01::","synset_id":"territory.n.03"},{"index":2,"sense_key":"territory%1:15:00::","synset_id":"district.n.01"},{"index":3,"sense_key":"territory%1:09:00::","synset_id":"territory.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"territory%1:15:01::","call_ids":["semeval2013.d011.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"territory%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.00413,"input_uncached_usd":0.0022700000000000003,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.601692209020257}
{"item_id":"semeval2013.d011.s007.t003","gold_sense_keys":["horde%1:14:00::"],"candidates":[{"index":1,"sense_key":"horde%1:14:00::","synset_id":"horde.n.01"},{"index":2,"sense_key":"horde%1:14:01::","synset_id":"drove.n.02"},{"index":3,"sense_key":"horde%1:14:02::","synset_id":"horde.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"horde%1:14:00::","call_ids":["semeval2013.d011.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"horde%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":413,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":109},"cost":{"currency":"USD","total_usd":0.005785,"input_uncached_usd":0.002065,"input_cached_usd":0.0,"output_usd":0.00372,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.247009666985832}
{"item_id":"semeval2013.d011.s007.t005","gold_sense_keys":["left%1:14:00::"],"candidates":[{"index":1,"sense_key":"left%1:15:00::","synset_id":"left.n.01"},{"index":2,"sense_key":"left%1:06:00::","synset_id":"left_field.n.01"},{"index":3,"sense_key":"left%1:08:00::","synset_id":"left.n.03"},{"index":4,"sense_key":"left%1:14:00::","synset_id":"left.n.02"},{"index":5,"sense_key":"left%1:04:00::","synset_id":"left.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"left%1:14:00::","call_ids":["semeval2013.d011.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"left%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":558,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.004110000000000001,"input_uncached_usd":0.0027900000000000004,"input_cached_usd":0.0,"output_usd":0.00132,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.06090145802591}
{"item_id":"semeval2013.d011.s007.t006","gold_sense_keys":["right%1:14:00::"],"candidates":[{"index":1,"sense_key":"right%1:06:00::","synset_id":"right_field.n.01"},{"index":2,"sense_key":"right%1:04:00::","synset_id":"right.n.06"},{"index":3,"sense_key":"right%1:15:00::","synset_id":"right.n.02"},{"index":4,"sense_key":"right%1:08:00::","synset_id":"right.n.05"},{"index":5,"sense_key":"right%1:21:00::","synset_id":"right.n.08"},{"index":6,"sense_key":"right%1:07:00::","synset_id":"right.n.01"},{"index":7,"sense_key":"right%1:07:01::","synset_id":"right.n.07"},{"index":8,"sense_key":"right%1:14:00::","synset_id":"right.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"right%1:14:00::","call_ids":["semeval2013.d011.s007.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"right%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.005115,"input_uncached_usd":0.0036450000000000002,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3986789170885459}
{"item_id":"semeval2013.d011.s008.t000","gold_sense_keys":["side%1:14:00::"],"candidates":[{"index":1,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":2,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":3,"sense_key":"side%1:11:00::","synset_id":"english.n.04"},{"index":4,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":5,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":6,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":7,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":8,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":9,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":10,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":11,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":12,"sense_key":"side%1:15:00::","synset_id":"side.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"side%1:14:00::","call_ids":["semeval2013.d011.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"side%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":822,"cached_input_tokens":0,"output_tokens":175,"reasoning_output_tokens":159},"cost":{"currency":"USD","total_usd":0.00936,"input_uncached_usd":0.00411,"input_cached_usd":0.0,"output_usd":0.00525,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.733540292014368}
{"item_id":"semeval2013.d011.s008.t002","gold_sense_keys":["burden%1:09:01::"],"candidates":[{"index":1,"sense_key":"burden%1:09:01::","synset_id":"burden.n.01"},{"index":2,"sense_key":"burden%1:09:00::","synset_id":"burden.n.04"},{"index":3,"sense_key":"burden%1:06:01::","synset_id":"load.n.01"},{"index":4,"sense_key":"burden%1:10:00::","synset_id":"effect.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"burden%1:09:01::","call_ids":["semeval2013.d011.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"burden%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.0038850000000000004,"input_uncached_usd":0.0023250000000000002,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.094750583055429}
{"item_id":"semeval2013.d011.s008.t003","gold_sense_keys":["economy%1:14:00::"],"candidates":[{"index":1,"sense_key":"economy%1:14:00::","synset_id":"economy.n.01"},{"index":2,"sense_key":"economy%1:07:00::","synset_id":"economy.n.03"},{"index":3,"sense_key":"economy%1:09:01::","synset_id":"economy.n.02"},{"index":4,"sense_key":"economy%1:04:00::","synset_id":"economy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"economy%1:14:00::","call_ids":["semeval2013.d011.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"economy%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.003955,"input_uncached_usd":0.0023650000000000003,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.389739332953468}
{"item_id":"semeval2013.d011.s008.t004","gold_sense_keys":["society%1:14:00::"],"candidates":[{"index":1,"sense_key":"society%1:14:01::","synset_id":"club.n.02"},{"index":2,"sense_key":"society%1:14:00::","synset_id":"society.n.01"},{"index":3,"sense_key":"society%1:14:02::","synset_id":"society.n.04"},{"index":4,"sense_key":"society%1:26:00::","synset_id":"company.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"society%1:14:00::","call_ids":["semeval2013.d011.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"society%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.003985,"input_uncached_usd":0.002395,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5044942090753466}
{"item_id":"semeval2013.d011.s009.t000","gold_sense_keys":["difference%1:07:00::"],"candidates":[{"index":1,"sense_key":"difference%1:11:00::","synset_id":"deviation.n.01"},{"index":2,"sense_key":"difference%1:24:00::","synset_id":"difference.n.04"},{"index":3,"sense_key":"difference%1:23:00::","synset_id":"remainder.n.03"},{"index":4,"sense_key":"difference%1:07:00::","synset_id":"difference.n.01"},{"index":5,"sense_key":"difference%1:10:00::","synset_id":"dispute.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"difference%1:07:00::","call_ids":["semeval2013.d011.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"difference%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.004750000000000001,"input_uncached_usd":0.00268,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.19806162500754}
{"item_id":"semeval2013.d011.s009.t001","gold_sense_keys":["degree%1:07:00::","degree%1:26:01::"],"candidates":[{"index":1,"sense_key":"degree%1:07:01::","synset_id":"degree.n.07"},{"index":2,"sense_key":"degree%1:26:01::","synset_id":"degree.n.02"},{"index":3,"sense_key":"degree%1:07:00::","synset_id":"degree.n.01"},{"index":4,"sense_key":"degree%1:10:00::","synset_id":"academic_degree.n.01"},{"index":5,"sense_key":"degree%1:09:00::","synset_id":"degree.n.05"},{"index":6,"sense_key":"degree%1:23:00::","synset_id":"degree.n.04"},{"index":7,"sense_key":"degree%1:23:03::","synset_id":"degree.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"degree%1:26:01::","call_ids":["semeval2013.d011.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"degree%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018995,"input_uncached_usd":0.0031550000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.294984041946009}
{"item_id":"semeval2013.d011.s009.t002","gold_sense_keys":["generosity%1:07:00::"],"candidates":[{"index":1,"sense_key":"generosity%1:07:00::","synset_id":"generosity.n.01"},{"index":2,"sense_key":"generosity%1:04:00::","synset_id":"generosity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"generosity%1:07:00::","call_ids":["semeval2013.d011.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"generosity%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017755,"input_uncached_usd":0.0019450000000000001,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.672300625010394}
{"item_id":"semeval2013.d011.s010.t000","gold_sense_keys":["program%1:10:04::"],"candidates":[{"index":1,"sense_key":"program%1:10:05::","synset_id":"course_of_study.n.01"},{"index":2,"sense_key":"program%1:10:01::","synset_id":"broadcast.n.02"},{"index":3,"sense_key":"program%1:09:01::","synset_id":"program.n.02"},{"index":4,"sense_key":"program%1:10:02::","synset_id":"program.n.07"},{"index":5,"sense_key":"program%1:09:00::","synset_id":"plan.n.01"},{"index":6,"sense_key":"program%1:04:00::","synset_id":"program.n.08"},{"index":7,"sense_key":"program%1:10:04::","synset_id":"platform.n.02"},{"index":8,"sense_key":"program%1:10:00::","synset_id":"program.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"program%1:10:04::","call_ids":["semeval2013.d011.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"program%1:10:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":726,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":115},"cost":{"currency":"USD","total_usd":0.007560000000000001,"input_uncached_usd":0.0036300000000000004,"input_cached_usd":0.0,"output_usd":0.00393,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2561757089570165}
{"item_id":"semeval2013.d011.s010.t003","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:26:00::","call_ids":["semeval2013.d011.s010.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01812,"input_uncached_usd":0.0022800000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.246408832957968}
{"item_id":"semeval2013.d011.s011.t000","gold_sense_keys":["notion%1:09:02::"],"candidates":[{"index":1,"sense_key":"notion%1:09:01::","synset_id":"notion.n.03"},{"index":2,"sense_key":"notion%1:09:02::","synset_id":"impression.n.01"},{"index":3,"sense_key":"notion%1:06:00::","synset_id":"notion.n.04"},{"index":4,"sense_key":"notion%1:09:00::","synset_id":"notion.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notion%1:09:02::","call_ids":["semeval2013.d011.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notion%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":230,"reasoning_output_tokens":214},"cost":{"currency":"USD","total_usd":0.009540000000000002,"input_uncached_usd":0.0026400000000000004,"input_cached_usd":0.0,"output_usd":0.0069,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.536045792046934}
{"item_id":"semeval2013.d011.s011.t002","gold_sense_keys":["burden%1:09:01::"],"candidates":[{"index":1,"sense_key":"burden%1:09:00::","synset_id":"burden.n.04"},{"index":2,"sense_key":"burden%1:06:01::","synset_id":"load.n.01"},{"index":3,"sense_key":"burden%1:10:00::","synset_id":"effect.n.04"},{"index":4,"sense_key":"burden%1:09:01::","synset_id":"burden.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"burden%1:09:01::","call_ids":["semeval2013.d011.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"burden%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.0046099999999999995,"input_uncached_usd":0.00251,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2328414580551907}
{"item_id":"semeval2013.d011.s011.t003","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"problem%1:09:00::","call_ids":["semeval2013.d011.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"problem%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01819,"input_uncached_usd":0.00235,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.893975415965542}
{"item_id":"semeval2013.d011.s012.t001","gold_sense_keys":["book%1:10:00::"],"candidates":[{"index":1,"sense_key":"book%1:10:00::","synset_id":"book.n.01"},{"index":2,"sense_key":"book%1:10:02::","synset_id":"record.n.05"},{"index":3,"sense_key":"book%1:14:00::","synset_id":"book.n.07"},{"index":4,"sense_key":"book%1:10:05::","synset_id":"koran.n.01"},{"index":5,"sense_key":"book%1:21:00::","synset_id":"ledger.n.01"},{"index":6,"sense_key":"book%1:14:01::","synset_id":"book.n.06"},{"index":7,"sense_key":"book%1:10:03::","synset_id":"book.n.10"},{"index":8,"sense_key":"book%1:06:00::","synset_id":"book.n.02"},{"index":9,"sense_key":"book%1:10:04::","synset_id":"bible.n.01"},{"index":10,"sense_key":"book%1:06:02::","synset_id":"book.n.11"},{"index":11,"sense_key":"book%1:10:01::","synset_id":"script.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"book%1:10:00::","call_ids":["semeval2013.d011.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"book%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":836,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":96},"cost":{"currency":"USD","total_usd":0.007510000000000001,"input_uncached_usd":0.0041800000000000006,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.451348875067197}
{"item_id":"semeval2013.d011.s012.t003","gold_sense_keys":["immigration%1:04:00::"],"candidates":[{"index":1,"sense_key":"immigration%1:04:00::","synset_id":"immigration.n.01"},{"index":2,"sense_key":"immigration%1:14:00::","synset_id":"immigration.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"immigration%1:04:00::","call_ids":["semeval2013.d011.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"immigration%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.004675,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2461271249921992}
{"item_id":"semeval2013.d011.s012.t004","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":2,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":3,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"},{"index":4,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"people%1:14:00::","call_ids":["semeval2013.d011.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":181,"reasoning_output_tokens":165},"cost":{"currency":"USD","total_usd":0.008085000000000002,"input_uncached_usd":0.0026550000000000002,"input_cached_usd":0.0,"output_usd":0.00543,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.971959125017747}
{"item_id":"semeval2013.d011.s012.t005","gold_sense_keys":["migration%1:04:00::"],"candidates":[{"index":1,"sense_key":"migration%1:04:00::","synset_id":"migration.n.01"},{"index":2,"sense_key":"migration%1:14:00::","synset_id":"migration.n.02"},{"index":3,"sense_key":"migration%1:11:01::","synset_id":"migration.n.03"},{"index":4,"sense_key":"migration%1:11:00::","synset_id":"migration.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"migration%1:04:00::","call_ids":["semeval2013.d011.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"migration%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004445,"input_uncached_usd":0.002435,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.483998249983415}
{"item_id":"semeval2013.d011.s012.t007","gold_sense_keys":["future%1:28:00::"],"candidates":[{"index":1,"sense_key":"future%1:10:00::","synset_id":"future.n.02"},{"index":2,"sense_key":"future%1:28:00::","synset_id":"future.n.01"},{"index":3,"sense_key":"future%1:06:00::","synset_id":"future.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"future%1:28:00::","call_ids":["semeval2013.d011.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"future%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.0044800000000000005,"input_uncached_usd":0.0022600000000000003,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.718407417065464}
{"item_id":"semeval2013.d011.s012.t009","gold_sense_keys":["specialist%1:18:00::"],"candidates":[{"index":1,"sense_key":"specialist%1:18:00::","synset_id":"specialist.n.01"},{"index":2,"sense_key":"specialist%1:18:01::","synset_id":"specialist.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"specialist%1:18:00::","call_ids":["semeval2013.d011.s012.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"specialist%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":423,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.004155000000000001,"input_uncached_usd":0.002115,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8598126249853522}
{"item_id":"semeval2013.d011.s012.t010","gold_sense_keys":["impact%1:19:00::"],"candidates":[{"index":1,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":2,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"},{"index":3,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":4,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"impact%1:19:00::","call_ids":["semeval2013.d011.s012.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"impact%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.00425,"input_uncached_usd":0.0025700000000000002,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.661297666025348}
{"item_id":"semeval2013.d011.s012.t011","gold_sense_keys":["migration%1:04:00::"],"candidates":[{"index":1,"sense_key":"migration%1:11:01::","synset_id":"migration.n.03"},{"index":2,"sense_key":"migration%1:14:00::","synset_id":"migration.n.02"},{"index":3,"sense_key":"migration%1:11:00::","synset_id":"migration.n.04"},{"index":4,"sense_key":"migration%1:04:00::","synset_id":"migration.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"migration%1:04:00::","call_ids":["semeval2013.d011.s012.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"migration%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":79},"cost":{"currency":"USD","total_usd":0.005255,"input_uncached_usd":0.002435,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.272693375009112}
{"item_id":"semeval2013.d011.s012.t012","gold_sense_keys":["flow%1:26:00::"],"candidates":[{"index":1,"sense_key":"flow%1:04:00::","synset_id":"flow.n.03"},{"index":2,"sense_key":"flow%1:14:00::","synset_id":"stream.n.02"},{"index":3,"sense_key":"flow%1:26:00::","synset_id":"stream.n.04"},{"index":4,"sense_key":"flow%1:11:00::","synset_id":"flow.n.01"},{"index":5,"sense_key":"flow%1:22:01::","synset_id":"menstruation.n.01"},{"index":6,"sense_key":"flow%1:28:00::","synset_id":"flow.n.02"},{"index":7,"sense_key":"flow%1:22:00::","synset_id":"flow.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"flow%1:26:00::","call_ids":["semeval2013.d011.s012.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"flow%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":241,"reasoning_output_tokens":225},"cost":{"currency":"USD","total_usd":0.01059,"input_uncached_usd":0.00336,"input_cached_usd":0.0,"output_usd":0.00723,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.548207166022621}
{"item_id":"semeval2013.d011.s012.t015","gold_sense_keys":["employment%1:04:00::","employment%1:04:02::","employment%1:26:00::"],"candidates":[{"index":1,"sense_key":"employment%1:26:00::","synset_id":"employment.n.01"},{"index":2,"sense_key":"employment%1:04:01::","synset_id":"use.n.01"},{"index":3,"sense_key":"employment%1:04:02::","synset_id":"employment.n.03"},{"index":4,"sense_key":"employment%1:04:00::","synset_id":"employment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"employment%1:26:00::","call_ids":["semeval2013.d011.s012.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"employment%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":190,"reasoning_output_tokens":175},"cost":{"currency":"USD","total_usd":0.00834,"input_uncached_usd":0.0026400000000000004,"input_cached_usd":0.0,"output_usd":0.0057,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.085472957929596}
{"item_id":"semeval2013.d011.s012.t016","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":2,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":5,"sense_key":"country%1:15:01::","synset_id":"country.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d011.s012.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":372,"reasoning_output_tokens":356},"cost":{"currency":"USD","total_usd":0.01417,"input_uncached_usd":0.00301,"input_cached_usd":0.0,"output_usd":0.01116,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.163881125045009}
{"item_id":"semeval2013.d011.s013.t000","gold_sense_keys":["immigration%1:04:00::"],"candidates":[{"index":1,"sense_key":"immigration%1:14:00::","synset_id":"immigration.n.02"},{"index":2,"sense_key":"immigration%1:04:00::","synset_id":"immigration.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"immigration%1:04:00::","call_ids":["semeval2013.d011.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"immigration%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.00452,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.209343208000064}
{"item_id":"semeval2013.d011.s013.t001","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":2,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":3,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":4,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":5,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":6,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"},{"index":7,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"growth%1:22:02::","call_ids":["semeval2013.d011.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"growth%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.00512,"input_uncached_usd":0.0032600000000000003,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.18059333297424}
{"item_id":"semeval2013.d011.s013.t002","gold_sense_keys":["employment%1:04:00::","employment%1:04:02::","employment%1:26:00::"],"candidates":[{"index":1,"sense_key":"employment%1:04:00::","synset_id":"employment.n.02"},{"index":2,"sense_key":"employment%1:04:01::","synset_id":"use.n.01"},{"index":3,"sense_key":"employment%1:04:02::","synset_id":"employment.n.03"},{"index":4,"sense_key":"employment%1:26:00::","synset_id":"employment.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"employment%1:26:00::","call_ids":["semeval2013.d011.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"employment%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":233,"reasoning_output_tokens":217},"cost":{"currency":"USD","total_usd":0.00965,"input_uncached_usd":0.00266,"input_cached_usd":0.0,"output_usd":0.0069900000000000006,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8305146250640973}
{"item_id":"semeval2013.d011.s014.t000","gold_sense_keys":["fear%1:12:01::"],"candidates":[{"index":1,"sense_key":"fear%1:12:00::","synset_id":"fear.n.01"},{"index":2,"sense_key":"fear%1:12:01::","synset_id":"concern.n.02"},{"index":3,"sense_key":"fear%1:12:02::","synset_id":"fear.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fear%1:12:01::","call_ids":["semeval2013.d011.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fear%1:12:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.005175,"input_uncached_usd":0.002595,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.174661415978335}
{"item_id":"semeval2013.d011.s014.t001","gold_sense_keys":["impact%1:19:00::"],"candidates":[{"index":1,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":2,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"},{"index":3,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":4,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"impact%1:19:00::","call_ids":["semeval2013.d011.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"impact%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":118},"cost":{"currency":"USD","total_usd":0.0067,"input_uncached_usd":0.00268,"input_cached_usd":0.0,"output_usd":0.00402,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1120507910382003}
{"item_id":"semeval2013.d011.s014.t003","gold_sense_keys":["notion%1:09:02::"],"candidates":[{"index":1,"sense_key":"notion%1:09:01::","synset_id":"notion.n.03"},{"index":2,"sense_key":"notion%1:09:00::","synset_id":"notion.n.02"},{"index":3,"sense_key":"notion%1:09:02::","synset_id":"impression.n.01"},{"index":4,"sense_key":"notion%1:06:00::","synset_id":"notion.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"notion%1:09:02::","call_ids":["semeval2013.d011.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"notion%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":554,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":86},"cost":{"currency":"USD","total_usd":0.00583,"input_uncached_usd":0.0027700000000000003,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.8000367500353605}
{"item_id":"semeval2013.d011.s014.t004","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"},{"index":2,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"},{"index":3,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"},{"index":4,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worker%1:18:00::","call_ids":["semeval2013.d011.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":541,"cached_input_tokens":0,"output_tokens":431,"reasoning_output_tokens":415},"cost":{"currency":"USD","total_usd":0.015635,"input_uncached_usd":0.0027050000000000004,"input_cached_usd":0.0,"output_usd":0.01293,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.187250250019133}
{"item_id":"semeval2013.d011.s014.t005","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:09:00::","synset_id":"one.n.02"},{"index":2,"sense_key":"one%1:23:00::","synset_id":"one.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"one%1:09:00::","call_ids":["semeval2013.d011.s014.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.00498,"input_uncached_usd":0.00237,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1051899169106036}
{"item_id":"semeval2013.d011.s014.t006","gold_sense_keys":["pressure%1:07:00::"],"candidates":[{"index":1,"sense_key":"pressure%1:19:00::","synset_id":"pressure.n.01"},{"index":2,"sense_key":"pressure%1:26:01::","synset_id":"pressure.n.06"},{"index":3,"sense_key":"pressure%1:09:00::","synset_id":"pressure.n.05"},{"index":4,"sense_key":"pressure%1:04:00::","synset_id":"press.n.09"},{"index":5,"sense_key":"pressure%1:19:01::","synset_id":"atmospheric_pressure.n.01"},{"index":6,"sense_key":"pressure%1:26:00::","synset_id":"imperativeness.n.01"},{"index":7,"sense_key":"pressure%1:07:00::","synset_id":"pressure.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"pressure%1:07:00::","call_ids":["semeval2013.d011.s014.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"pressure%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":225,"reasoning_output_tokens":209},"cost":{"currency":"USD","total_usd":0.010205,"input_uncached_usd":0.003455,"input_cached_usd":0.0,"output_usd":0.00675,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.303636832977645}
{"item_id":"semeval2013.d011.s015.t000","gold_sense_keys":["experience%1:09:01::"],"candidates":[{"index":1,"sense_key":"experience%1:09:00::","synset_id":"experience.n.02"},{"index":2,"sense_key":"experience%1:11:00::","synset_id":"experience.n.03"},{"index":3,"sense_key":"experience%1:09:01::","synset_id":"experience.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"experience%1:09:01::","call_ids":["semeval2013.d011.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"experience%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.004775000000000001,"input_uncached_usd":0.0025550000000000004,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.905782709014602}
{"item_id":"semeval2013.d011.s015.t003","gold_sense_keys":["replacement%1:09:00::","replacement%1:18:02::"],"candidates":[{"index":1,"sense_key":"replacement%1:04:00::","synset_id":"replacement.n.01"},{"index":2,"sense_key":"replacement%1:11:00::","synset_id":"substitution.n.01"},{"index":3,"sense_key":"replacement%1:18:01::","synset_id":"successor.n.01"},{"index":4,"sense_key":"replacement%1:18:02::","synset_id":"surrogate.n.01"},{"index":5,"sense_key":"replacement%1:09:00::","synset_id":"substitute.n.01"},{"index":6,"sense_key":"replacement%1:22:00::","synset_id":"refilling.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"replacement%1:09:00::","call_ids":["semeval2013.d011.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"replacement%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":322,"reasoning_output_tokens":306},"cost":{"currency":"USD","total_usd":0.0128,"input_uncached_usd":0.0031400000000000004,"input_cached_usd":0.0,"output_usd":0.00966,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.898493250017054}
{"item_id":"semeval2013.d011.s015.t004","gold_sense_keys":["labor%1:14:00::"],"candidates":[{"index":1,"sense_key":"labor%1:04:01::","synset_id":"undertaking.n.01"},{"index":2,"sense_key":"labor%1:14:00::","synset_id":"labor.n.01"},{"index":3,"sense_key":"labor%1:04:00::","synset_id":"labor.n.02"},{"index":4,"sense_key":"labor%1:14:04::","synset_id":"labor_movement.n.01"},{"index":5,"sense_key":"labor%1:14:02::","synset_id":"british_labour_party.n.01"},{"index":6,"sense_key":"labor%1:14:01::","synset_id":"department_of_labor.n.01"},{"index":7,"sense_key":"labor%1:26:00::","synset_id":"parturiency.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"labor%1:14:00::","call_ids":["semeval2013.d011.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"labor%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":194,"reasoning_output_tokens":178},"cost":{"currency":"USD","total_usd":0.009585,"input_uncached_usd":0.003765,"input_cached_usd":0.0,"output_usd":0.0058200000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.285325167002156}
{"item_id":"semeval2013.d011.s015.t005","gold_sense_keys":["trade%1:04:00::"],"candidates":[{"index":1,"sense_key":"trade%1:04:00::","synset_id":"trade.n.02"},{"index":2,"sense_key":"trade%1:04:02::","synset_id":"deal.n.01"},{"index":3,"sense_key":"trade%1:04:05::","synset_id":"trade.n.01"},{"index":4,"sense_key":"trade%1:04:01::","synset_id":"trade.n.03"},{"index":5,"sense_key":"trade%1:19:00::","synset_id":"trade_wind.n.01"},{"index":6,"sense_key":"trade%1:04:03::","synset_id":"barter.n.01"},{"index":7,"sense_key":"trade%1:14:00::","synset_id":"craft.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trade%1:04:00::","call_ids":["semeval2013.d011.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trade%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":716,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":95},"cost":{"currency":"USD","total_usd":0.00691,"input_uncached_usd":0.0035800000000000003,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5603813339257613}
{"item_id":"semeval2013.d011.s015.t006","gold_sense_keys":["shortage%1:26:00::"],"candidates":[{"index":1,"sense_key":"shortage%1:26:00::","synset_id":"dearth.n.01"},{"index":2,"sense_key":"shortage%1:07:00::","synset_id":"deficit.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"shortage%1:26:00::","call_ids":["semeval2013.d011.s015.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"shortage%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":137},"cost":{"currency":"USD","total_usd":0.0069700000000000005,"input_uncached_usd":0.0024100000000000002,"input_cached_usd":0.0,"output_usd":0.00456,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.904853000072762}
{"item_id":"semeval2013.d011.s015.t007","gold_sense_keys":["labor%1:14:00::"],"candidates":[{"index":1,"sense_key":"labor%1:14:02::","synset_id":"british_labour_party.n.01"},{"index":2,"sense_key":"labor%1:14:00::","synset_id":"labor.n.01"},{"index":3,"sense_key":"labor%1:04:01::","synset_id":"undertaking.n.01"},{"index":4,"sense_key":"labor%1:14:01::","synset_id":"department_of_labor.n.01"},{"index":5,"sense_key":"labor%1:04:00::","synset_id":"labor.n.02"},{"index":6,"sense_key":"labor%1:26:00::","synset_id":"parturiency.n.01"},{"index":7,"sense_key":"labor%1:14:04::","synset_id":"labor_movement.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"labor%1:14:00::","call_ids":["semeval2013.d011.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"labor%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":112},"cost":{"currency":"USD","total_usd":0.007605000000000001,"input_uncached_usd":0.003765,"input_cached_usd":0.0,"output_usd":0.00384,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.9341749580344185}
{"item_id":"semeval2013.d011.s016.t001","gold_sense_keys":["sector%1:14:00::"],"candidates":[{"index":1,"sense_key":"sector%1:25:00::","synset_id":"sector.n.01"},{"index":2,"sense_key":"sector%1:15:00::","synset_id":"sector.n.05"},{"index":3,"sense_key":"sector%1:06:00::","synset_id":"sector.n.06"},{"index":4,"sense_key":"sector%1:09:00::","synset_id":"sector.n.03"},{"index":5,"sense_key":"sector%1:23:00::","synset_id":"sector.n.04"},{"index":6,"sense_key":"sector%1:14:00::","synset_id":"sector.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"sector%1:14:00::","call_ids":["semeval2013.d011.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"sector%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":104},"cost":{"currency":"USD","total_usd":0.006445,"input_uncached_usd":0.0028450000000000003,"input_cached_usd":0.0,"output_usd":0.0036,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3880694159306586}
{"item_id":"semeval2013.d011.s016.t002","gold_sense_keys":["labor%1:14:00::"],"candidates":[{"index":1,"sense_key":"labor%1:04:00::","synset_id":"labor.n.02"},{"index":2,"sense_key":"labor%1:14:00::","synset_id":"labor.n.01"},{"index":3,"sense_key":"labor%1:14:02::","synset_id":"british_labour_party.n.01"},{"index":4,"sense_key":"labor%1:14:01::","synset_id":"department_of_labor.n.01"},{"index":5,"sense_key":"labor%1:04:01::","synset_id":"undertaking.n.01"},{"index":6,"sense_key":"labor%1:26:00::","synset_id":"parturiency.n.01"},{"index":7,"sense_key":"labor%1:14:04::","synset_id":"labor_movement.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"labor%1:14:00::","call_ids":["semeval2013.d011.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"labor%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":98},"cost":{"currency":"USD","total_usd":0.007095000000000001,"input_uncached_usd":0.0036750000000000003,"input_cached_usd":0.0,"output_usd":0.0034200000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5255448329262435}
{"item_id":"semeval2013.d011.s016.t003","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["semeval2013.d011.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":81},"cost":{"currency":"USD","total_usd":0.005235,"input_uncached_usd":0.0023550000000000003,"input_cached_usd":0.0,"output_usd":0.00288,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6064618750242516}
{"item_id":"semeval2013.d011.s016.t004","gold_sense_keys":["sector%1:14:00::"],"candidates":[{"index":1,"sense_key":"sector%1:14:00::","synset_id":"sector.n.02"},{"index":2,"sense_key":"sector%1:15:00::","synset_id":"sector.n.05"},{"index":3,"sense_key":"sector%1:06:00::","synset_id":"sector.n.06"},{"index":4,"sense_key":"sector%1:25:00::","synset_id":"sector.n.01"},{"index":5,"sense_key":"sector%1:09:00::","synset_id":"sector.n.03"},{"index":6,"sense_key":"sector%1:23:00::","synset_id":"sector.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sector%1:14:00::","call_ids":["semeval2013.d011.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sector%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":110},"cost":{"currency":"USD","total_usd":0.0066,"input_uncached_usd":0.00285,"input_cached_usd":0.0,"output_usd":0.0037500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2739689589943737}
{"item_id":"semeval2013.d011.s016.t005","gold_sense_keys":["training%1:04:00::"],"candidates":[{"index":1,"sense_key":"training%1:07:00::","synset_id":"education.n.05"},{"index":2,"sense_key":"training%1:04:00::","synset_id":"training.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"training%1:04:00::","call_ids":["semeval2013.d011.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"training%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":103},"cost":{"currency":"USD","total_usd":0.005795,"input_uncached_usd":0.002225,"input_cached_usd":0.0,"output_usd":0.0035700000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.060107958037406}
{"item_id":"semeval2013.d011.s016.t006","gold_sense_keys":["offer%1:10:00::"],"candidates":[{"index":1,"sense_key":"offer%1:10:01::","synset_id":"offer.n.01"},{"index":2,"sense_key":"offer%1:10:00::","synset_id":"offer.n.02"},{"index":3,"sense_key":"offer%1:04:01::","synset_id":"crack.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"offer%1:10:00::","call_ids":["semeval2013.d011.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"offer%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":338,"reasoning_output_tokens":322},"cost":{"currency":"USD","total_usd":0.012629999999999999,"input_uncached_usd":0.00249,"input_cached_usd":0.0,"output_usd":0.01014,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.273173874942586}
{"item_id":"semeval2013.d011.s016.t007","gold_sense_keys":["employment%1:04:00::","employment%1:26:00::"],"candidates":[{"index":1,"sense_key":"employment%1:04:00::","synset_id":"employment.n.02"},{"index":2,"sense_key":"employment%1:04:01::","synset_id":"use.n.01"},{"index":3,"sense_key":"employment%1:26:00::","synset_id":"employment.n.01"},{"index":4,"sense_key":"employment%1:04:02::","synset_id":"employment.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"employment%1:04:00::","call_ids":["semeval2013.d011.s016.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"employment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.034104999999999996,"input_uncached_usd":0.0027250000000000004,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":16.013061083038338}
{"item_id":"semeval2013.d011.s017.t000","gold_sense_keys":["effect%1:19:00::"],"candidates":[{"index":1,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"},{"index":2,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"},{"index":3,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":4,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"},{"index":5,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":6,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"effect%1:19:00::","call_ids":["semeval2013.d011.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"effect%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.005455,"input_uncached_usd":0.003295,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.185590457986109}
{"item_id":"semeval2013.d011.s018.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":2,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":3,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":6,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":7,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":8,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":9,"sense_key":"way%1:21:00::","synset_id":"way.n.12"},{"index":10,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":11,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":12,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"way%1:07:01::","call_ids":["semeval2013.d011.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.007255000000000001,"input_uncached_usd":0.004135000000000001,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.696674917009659}
{"item_id":"semeval2013.d011.s018.t002","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":2,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":3,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":4,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":5,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"country%1:15:00::","call_ids":["semeval2013.d011.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"country%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":147},"cost":{"currency":"USD","total_usd":0.007915,"input_uncached_usd":0.0030250000000000003,"input_cached_usd":0.0,"output_usd":0.00489,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.971369666978717}
{"item_id":"semeval2013.d011.s018.t003","gold_sense_keys":["shortage%1:26:00::"],"candidates":[{"index":1,"sense_key":"shortage%1:07:00::","synset_id":"deficit.n.01"},{"index":2,"sense_key":"shortage%1:26:00::","synset_id":"dearth.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"shortage%1:26:00::","call_ids":["semeval2013.d011.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"shortage%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":212,"reasoning_output_tokens":196},"cost":{"currency":"USD","total_usd":0.008595,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.00636,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.615232917014509}
{"item_id":"semeval2013.d011.s018.t005","gold_sense_keys":["complementarity%1:24:02::"],"candidates":[{"index":1,"sense_key":"complementarity%1:24:02::","synset_id":"complementarity.n.02"},{"index":2,"sense_key":"complementarity%1:24:01::","synset_id":"complementarity.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"complementarity%1:24:02::","call_ids":["semeval2013.d011.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"complementarity%1:24:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.00366,"input_uncached_usd":0.00216,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.687417499953881}
{"item_id":"semeval2013.d011.s018.t007","gold_sense_keys":["arrival%1:04:01::"],"candidates":[{"index":1,"sense_key":"arrival%1:18:00::","synset_id":"arrival.n.03"},{"index":2,"sense_key":"arrival%1:04:01::","synset_id":"arrival.n.02"},{"index":3,"sense_key":"arrival%1:04:00::","synset_id":"arrival.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arrival%1:04:01::","call_ids":["semeval2013.d011.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arrival%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.0042450000000000005,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.680516666965559}
{"item_id":"semeval2013.d011.s018.t009","gold_sense_keys":["effect%1:19:00::"],"candidates":[{"index":1,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"},{"index":2,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":3,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":4,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"},{"index":5,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"},{"index":6,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"effect%1:19:00::","call_ids":["semeval2013.d011.s018.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"effect%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.004705,"input_uncached_usd":0.003085,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6593395000090823}
{"item_id":"semeval2013.d011.s019.t000","gold_sense_keys":["rise%1:11:00::"],"candidates":[{"index":1,"sense_key":"rise%1:04:01::","synset_id":"advance.n.06"},{"index":2,"sense_key":"rise%1:11:03::","synset_id":"emanation.n.03"},{"index":3,"sense_key":"rise%1:11:02::","synset_id":"rise.n.04"},{"index":4,"sense_key":"rise%1:07:01::","synset_id":"upgrade.n.04"},{"index":5,"sense_key":"rise%1:17:00::","synset_id":"ascent.n.01"},{"index":6,"sense_key":"rise%1:07:03::","synset_id":"rise.n.09"},{"index":7,"sense_key":"rise%1:04:00::","synset_id":"rise.n.02"},{"index":8,"sense_key":"rise%1:11:00::","synset_id":"rise.n.01"},{"index":9,"sense_key":"rise%1:11:01::","synset_id":"lift.n.04"},{"index":10,"sense_key":"rise%1:07:02::","synset_id":"raise.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"rise%1:11:00::","call_ids":["semeval2013.d011.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"rise%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":810,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.006870000000000001,"input_uncached_usd":0.004050000000000001,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6277095419354737}
{"item_id":"semeval2013.d011.s019.t001","gold_sense_keys":["flow%1:26:00::"],"candidates":[{"index":1,"sense_key":"flow%1:26:00::","synset_id":"stream.n.04"},{"index":2,"sense_key":"flow%1:04:00::","synset_id":"flow.n.03"},{"index":3,"sense_key":"flow%1:14:00::","synset_id":"stream.n.02"},{"index":4,"sense_key":"flow%1:22:01::","synset_id":"menstruation.n.01"},{"index":5,"sense_key":"flow%1:22:00::","synset_id":"flow.n.04"},{"index":6,"sense_key":"flow%1:11:00::","synset_id":"flow.n.01"},{"index":7,"sense_key":"flow%1:28:00::","synset_id":"flow.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"flow%1:26:00::","call_ids":["semeval2013.d011.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"flow%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":719,"cached_input_tokens":0,"output_tokens":346,"reasoning_output_tokens":330},"cost":{"currency":"USD","total_usd":0.013975000000000001,"input_uncached_usd":0.003595,"input_cached_usd":0.0,"output_usd":0.01038,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.834228542051278}
{"item_id":"semeval2013.d011.s019.t002","gold_sense_keys":["increase%1:07:00::","increase%1:11:00::"],"candidates":[{"index":1,"sense_key":"increase%1:04:00::","synset_id":"increase.n.05"},{"index":2,"sense_key":"increase%1:11:00::","synset_id":"increase.n.02"},{"index":3,"sense_key":"increase%1:07:00::","synset_id":"increase.n.04"},{"index":4,"sense_key":"increase%1:22:00::","synset_id":"increase.n.03"},{"index":5,"sense_key":"increase%1:23:00::","synset_id":"addition.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"increase%1:07:00::","call_ids":["semeval2013.d011.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"increase%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":178,"reasoning_output_tokens":162},"cost":{"currency":"USD","total_usd":0.008395,"input_uncached_usd":0.0030550000000000004,"input_cached_usd":0.0,"output_usd":0.00534,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1022552909562364}
{"item_id":"semeval2013.d011.s020.t000","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"},{"index":2,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":3,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":4,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fact%1:09:01::","call_ids":["semeval2013.d011.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fact%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":562,"cached_input_tokens":0,"output_tokens":325,"reasoning_output_tokens":309},"cost":{"currency":"USD","total_usd":0.01256,"input_uncached_usd":0.0028100000000000004,"input_cached_usd":0.0,"output_usd":0.00975,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.45632595801726}
{"item_id":"semeval2013.d011.s020.t001","gold_sense_keys":["diversity%1:07:02::"],"candidates":[{"index":1,"sense_key":"diversity%1:07:02::","synset_id":"diverseness.n.01"},{"index":2,"sense_key":"diversity%1:07:01::","synset_id":"diversity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"diversity%1:07:02::","call_ids":["semeval2013.d011.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"diversity%1:07:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.004505,"input_uncached_usd":0.0023450000000000003,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.654060791945085}
{"item_id":"semeval2013.d011.s020.t003","gold_sense_keys":["creation%1:04:00::"],"candidates":[{"index":1,"sense_key":"creation%1:17:00::","synset_id":"universe.n.01"},{"index":2,"sense_key":"creation%1:11:00::","synset_id":"creation.n.03"},{"index":3,"sense_key":"creation%1:04:00::","synset_id":"creation.n.01"},{"index":4,"sense_key":"creation%1:06:00::","synset_id":"creation.n.02"},{"index":5,"sense_key":"creation%1:04:01::","synset_id":"initiation.n.02"},{"index":6,"sense_key":"creation%1:04:02::","synset_id":"creation.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"creation%1:04:00::","call_ids":["semeval2013.d011.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"creation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":195,"reasoning_output_tokens":179},"cost":{"currency":"USD","total_usd":0.009005,"input_uncached_usd":0.0031550000000000003,"input_cached_usd":0.0,"output_usd":0.00585,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.994370750035159}
{"item_id":"semeval2013.d011.s020.t004","gold_sense_keys":["idea%1:09:00::"],"candidates":[{"index":1,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"},{"index":2,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"},{"index":3,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"},{"index":4,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"},{"index":5,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"idea%1:09:00::","call_ids":["semeval2013.d011.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"idea%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":632,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.005350000000000001,"input_uncached_usd":0.00316,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.186998082906939}
{"item_id":"semeval2013.d011.s020.t006","gold_sense_keys":["proportion%1:24:01::"],"candidates":[{"index":1,"sense_key":"proportion%1:07:00::","synset_id":"proportion.n.02"},{"index":2,"sense_key":"proportion%1:25:00::","synset_id":"symmetry.n.02"},{"index":3,"sense_key":"proportion%1:24:01::","synset_id":"proportion.n.04"},{"index":4,"sense_key":"proportion%1:07:01::","synset_id":"proportion.n.05"},{"index":5,"sense_key":"proportion%1:24:00::","synset_id":"proportion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"proportion%1:24:00::","call_ids":["semeval2013.d011.s020.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"proportion%1:24:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":7377,"reasoning_output_tokens":7277},"cost":{"currency":"USD","total_usd":0.224435,"input_uncached_usd":0.003125,"input_cached_usd":0.0,"output_usd":0.22131,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":149.00429750001058}
{"item_id":"semeval2013.d011.s020.t008","gold_sense_keys":["american%3:01:01::"],"candidates":[{"index":1,"sense_key":"american%3:01:01::","synset_id":"american.a.01"},{"index":2,"sense_key":"american%3:01:00::","synset_id":"american.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%3:01:01::","call_ids":["semeval2013.d011.s020.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.004425,"input_uncached_usd":0.0023550000000000003,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7960197500651702}
{"item_id":"semeval2013.d011.s020.t010","gold_sense_keys":["winner%1:18:00::"],"candidates":[{"index":1,"sense_key":"winner%1:18:00::","synset_id":"winner.n.01"},{"index":2,"sense_key":"winner%1:18:01::","synset_id":"winner.n.02"},{"index":3,"sense_key":"winner%1:18:02::","synset_id":"achiever.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"winner%1:18:00::","call_ids":["semeval2013.d011.s020.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"winner%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":131},"cost":{"currency":"USD","total_usd":0.006865000000000001,"input_uncached_usd":0.002455,"input_cached_usd":0.0,"output_usd":0.00441,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.04032045789063}
{"item_id":"semeval2013.d011.s020.t012","gold_sense_keys":["yahoo%1:10:00::"],"candidates":[{"index":1,"sense_key":"yahoo%1:18:01::","synset_id":"yokel.n.01"},{"index":2,"sense_key":"yahoo%1:18:00::","synset_id":"yahoo.n.02"},{"index":3,"sense_key":"yahoo%1:10:00::","synset_id":"yahoo.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"yahoo%1:10:00::","call_ids":["semeval2013.d011.s020.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"yahoo%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.004030000000000001,"input_uncached_usd":0.00256,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.980539707932621}
{"item_id":"semeval2013.d011.s021.t001","gold_sense_keys":["contributor%1:18:00::"],"candidates":[{"index":1,"sense_key":"contributor%1:18:00::","synset_id":"subscriber.n.03"},{"index":2,"sense_key":"contributor%1:18:01::","synset_id":"contributor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contributor%1:18:00::","call_ids":["semeval2013.d011.s021.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contributor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.00494,"input_uncached_usd":0.00236,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.001041542040184}
{"item_id":"semeval2013.d011.s021.t003","gold_sense_keys":["budget%1:21:02::"],"candidates":[{"index":1,"sense_key":"budget%1:21:02::","synset_id":"budget.n.01"},{"index":2,"sense_key":"budget%1:21:03::","synset_id":"budget.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"budget%1:21:02::","call_ids":["semeval2013.d011.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"budget%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":235,"reasoning_output_tokens":220},"cost":{"currency":"USD","total_usd":0.009465,"input_uncached_usd":0.002415,"input_cached_usd":0.0,"output_usd":0.00705,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.896496208035387}
{"item_id":"semeval2013.d011.s022.t001","gold_sense_keys":["labor%1:14:00::"],"candidates":[{"index":1,"sense_key":"labor%1:14:02::","synset_id":"british_labour_party.n.01"},{"index":2,"sense_key":"labor%1:14:00::","synset_id":"labor.n.01"},{"index":3,"sense_key":"labor%1:14:04::","synset_id":"labor_movement.n.01"},{"index":4,"sense_key":"labor%1:04:00::","synset_id":"labor.n.02"},{"index":5,"sense_key":"labor%1:04:01::","synset_id":"undertaking.n.01"},{"index":6,"sense_key":"labor%1:26:00::","synset_id":"parturiency.n.01"},{"index":7,"sense_key":"labor%1:14:01::","synset_id":"department_of_labor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"labor%1:14:00::","call_ids":["semeval2013.d011.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"labor%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":2574,"reasoning_output_tokens":2535},"cost":{"currency":"USD","total_usd":0.08098500000000002,"input_uncached_usd":0.003765,"input_cached_usd":0.0,"output_usd":0.07722,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":43.97698779194616}
{"item_id":"semeval2013.d011.s022.t002","gold_sense_keys":["organisation%1:14:00::","organisation%1:14:01::"],"candidates":[{"index":1,"sense_key":"organisation%1:07:00::","synset_id":"organization.n.05"},{"index":2,"sense_key":"organisation%1:09:00::","synset_id":"arrangement.n.03"},{"index":3,"sense_key":"organisation%1:04:00::","synset_id":"organization.n.04"},{"index":4,"sense_key":"organisation%1:14:00::","synset_id":"organization.n.01"},{"index":5,"sense_key":"organisation%1:04:01::","synset_id":"organization.n.06"},{"index":6,"sense_key":"organisation%1:14:01::","synset_id":"administration.n.02"},{"index":7,"sense_key":"organisation%1:04:02::","synset_id":"constitution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"organisation%1:14:00::","call_ids":["semeval2013.d011.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"organisation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.00659,"input_uncached_usd":0.0038300000000000005,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.537529250024818}
{"item_id":"semeval2013.d011.s022.t006","gold_sense_keys":["age%1:28:00::"],"candidates":[{"index":1,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"},{"index":2,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"},{"index":3,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":4,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":5,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"age%1:28:00::","call_ids":["semeval2013.d011.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"age%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004855,"input_uncached_usd":0.0030550000000000004,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4915440829936415}
{"item_id":"semeval2013.d011.s022.t007","gold_sense_keys":["contribution%1:21:01::"],"candidates":[{"index":1,"sense_key":"contribution%1:04:01::","synset_id":"contribution.n.05"},{"index":2,"sense_key":"contribution%1:21:01::","synset_id":"contribution.n.04"},{"index":3,"sense_key":"contribution%1:21:00::","synset_id":"contribution.n.02"},{"index":4,"sense_key":"contribution%1:04:00::","synset_id":"contribution.n.03"},{"index":5,"sense_key":"contribution%1:04:02::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"contribution%1:21:01::","call_ids":["semeval2013.d011.s022.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"contribution%1:21:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.00651,"input_uncached_usd":0.0029700000000000004,"input_cached_usd":0.0,"output_usd":0.00354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.620658416999504}
{"item_id":"semeval2013.d011.s022.t009","gold_sense_keys":["expenditure%1:21:00::"],"candidates":[{"index":1,"sense_key":"expenditure%1:21:00::","synset_id":"outgo.n.01"},{"index":2,"sense_key":"expenditure%1:04:00::","synset_id":"expending.n.01"},{"index":3,"sense_key":"expenditure%1:04:01::","synset_id":"consumption.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"expenditure%1:21:00::","call_ids":["semeval2013.d011.s022.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"expenditure%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.0046300000000000004,"input_uncached_usd":0.0024400000000000003,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5751102089416236}
{"item_id":"semeval2013.d011.s022.t010","gold_sense_keys":["average%1:09:00::"],"candidates":[{"index":1,"sense_key":"average%1:09:01::","synset_id":"average.n.03"},{"index":2,"sense_key":"average%1:24:00::","synset_id":"average.n.02"},{"index":3,"sense_key":"average%1:09:00::","synset_id":"average.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"average%1:09:00::","call_ids":["semeval2013.d011.s022.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"average%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":90},"cost":{"currency":"USD","total_usd":0.005665,"input_uncached_usd":0.0024850000000000002,"input_cached_usd":0.0,"output_usd":0.00318,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.246856709010899}
{"item_id":"semeval2013.d011.s022.t012","gold_sense_keys":["budget%1:21:02::"],"candidates":[{"index":1,"sense_key":"budget%1:21:03::","synset_id":"budget.n.02"},{"index":2,"sense_key":"budget%1:21:02::","synset_id":"budget.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"budget%1:21:02::","call_ids":["semeval2013.d011.s022.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"budget%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":358,"reasoning_output_tokens":343},"cost":{"currency":"USD","total_usd":0.013090000000000001,"input_uncached_usd":0.00235,"input_cached_usd":0.0,"output_usd":0.01074,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.64425187499728}
{"item_id":"semeval2013.d011.s023.t001","gold_sense_keys":["mandate%1:04:00::"],"candidates":[{"index":1,"sense_key":"mandate%1:15:00::","synset_id":"mandate.n.02"},{"index":2,"sense_key":"mandate%1:04:00::","synset_id":"mandate.n.03"},{"index":3,"sense_key":"mandate%1:10:00::","synset_id":"mandate.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mandate%1:04:00::","call_ids":["semeval2013.d011.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mandate%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":80},"cost":{"currency":"USD","total_usd":0.00549,"input_uncached_usd":0.0026100000000000003,"input_cached_usd":0.0,"output_usd":0.00288,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3155723330564797}
{"item_id":"semeval2013.d011.s023.t003","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":2,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"},{"index":3,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":4,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":5,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":6,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":7,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"growth%1:22:02::","call_ids":["semeval2013.d011.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"growth%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00529,"input_uncached_usd":0.0034900000000000005,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.989948083064519}
{"item_id":"semeval2013.d011.s024.t000","gold_sense_keys":["question%1:10:01::"],"candidates":[{"index":1,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":2,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"},{"index":3,"sense_key":"question%1:10:03::","synset_id":"question.n.03"},{"index":4,"sense_key":"question%1:10:04::","synset_id":"question.n.06"},{"index":5,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"},{"index":6,"sense_key":"question%1:10:00::","synset_id":"question.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"question%1:10:01::","call_ids":["semeval2013.d011.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"question%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":81},"cost":{"currency":"USD","total_usd":0.006125,"input_uncached_usd":0.003245,"input_cached_usd":0.0,"output_usd":0.00288,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.724526625010185}
{"item_id":"semeval2013.d011.s024.t002","gold_sense_keys":["effect%1:19:00::"],"candidates":[{"index":1,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":2,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"},{"index":3,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"},{"index":4,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"},{"index":5,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":6,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"effect%1:19:00::","call_ids":["semeval2013.d011.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"effect%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.004855,"input_uncached_usd":0.003085,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8697793750325218}
{"item_id":"semeval2013.d012.s000.t000","gold_sense_keys":["pound%1:23:01::"],"candidates":[{"index":1,"sense_key":"pound%1:10:00::","synset_id":"pound.n.12"},{"index":2,"sense_key":"pound%1:23:05::","synset_id":"lebanese_pound.n.01"},{"index":3,"sense_key":"pound%1:18:00::","synset_id":"pound.n.11"},{"index":4,"sense_key":"pound%1:23:04::","synset_id":"irish_pound.n.01"},{"index":5,"sense_key":"pound%1:23:01::","synset_id":"british_pound.n.01"},{"index":6,"sense_key":"pound%1:23:02::","synset_id":"cypriot_pound.n.01"},{"index":7,"sense_key":"pound%1:23:08::","synset_id":"syrian_pound.n.01"},{"index":8,"sense_key":"pound%1:04:00::","synset_id":"hammer.n.08"},{"index":9,"sense_key":"pound%1:23:09::","synset_id":"pound.n.01"},{"index":10,"sense_key":"pound%1:06:00::","synset_id":"pound.n.13"},{"index":11,"sense_key":"pound%1:23:03::","synset_id":"egyptian_pound.n.01"},{"index":12,"sense_key":"pound%1:23:10::","synset_id":"pound.n.03"},{"index":13,"sense_key":"pound%1:23:00::","synset_id":"pound.n.10"},{"index":14,"sense_key":"pound%1:23:07::","synset_id":"sudanese_pound.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"pound%1:23:01::","call_ids":["semeval2013.d012.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"pound%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":782,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":80},"cost":{"currency":"USD","total_usd":0.00676,"input_uncached_usd":0.00391,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.302144958055578}
{"item_id":"semeval2013.d012.s000.t002","gold_sense_keys":["usa%1:15:00::"],"candidates":[{"index":1,"sense_key":"usa%1:14:00::","synset_id":"united_states_army.n.01"},{"index":2,"sense_key":"usa%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"usa%1:15:00::","call_ids":["semeval2013.d012.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"usa%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":312,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.003,"input_uncached_usd":0.0015600000000000002,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4210339169949293}
{"item_id":"semeval2013.d012.s001.t000","gold_sense_keys":["trouble%1:09:00::","trouble%1:11:00::","trouble%1:12:03::"],"candidates":[{"index":1,"sense_key":"trouble%1:11:00::","synset_id":"trouble.n.03"},{"index":2,"sense_key":"trouble%1:12:03::","synset_id":"worry.n.02"},{"index":3,"sense_key":"trouble%1:26:02::","synset_id":"trouble.n.06"},{"index":4,"sense_key":"trouble%1:09:00::","synset_id":"trouble.n.01"},{"index":5,"sense_key":"trouble%1:11:01::","synset_id":"fuss.n.02"},{"index":6,"sense_key":"trouble%1:04:02::","synset_id":"trouble.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trouble%1:09:00::","call_ids":["semeval2013.d012.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trouble%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":358,"reasoning_output_tokens":342},"cost":{"currency":"USD","total_usd":0.013205,"input_uncached_usd":0.002465,"input_cached_usd":0.0,"output_usd":0.01074,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.984830916975625}
{"item_id":"semeval2013.d012.s001.t001","gold_sense_keys":["uproar%1:26:00::"],"candidates":[{"index":1,"sense_key":"uproar%1:26:00::","synset_id":"tumult.n.01"},{"index":2,"sense_key":"uproar%1:11:00::","synset_id":"hubbub.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"uproar%1:26:00::","call_ids":["semeval2013.d012.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"uproar%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":333,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":96},"cost":{"currency":"USD","total_usd":0.004995,"input_uncached_usd":0.001665,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1877172919921577}
{"item_id":"semeval2013.d012.s001.t003","gold_sense_keys":["pound%1:23:01::"],"candidates":[{"index":1,"sense_key":"pound%1:23:09::","synset_id":"pound.n.01"},{"index":2,"sense_key":"pound%1:23:03::","synset_id":"egyptian_pound.n.01"},{"index":3,"sense_key":"pound%1:06:00::","synset_id":"pound.n.13"},{"index":4,"sense_key":"pound%1:23:01::","synset_id":"british_pound.n.01"},{"index":5,"sense_key":"pound%1:23:08::","synset_id":"syrian_pound.n.01"},{"index":6,"sense_key":"pound%1:18:00::","synset_id":"pound.n.11"},{"index":7,"sense_key":"pound%1:23:10::","synset_id":"pound.n.03"},{"index":8,"sense_key":"pound%1:10:00::","synset_id":"pound.n.12"},{"index":9,"sense_key":"pound%1:23:04::","synset_id":"irish_pound.n.01"},{"index":10,"sense_key":"pound%1:23:07::","synset_id":"sudanese_pound.n.01"},{"index":11,"sense_key":"pound%1:23:00::","synset_id":"pound.n.10"},{"index":12,"sense_key":"pound%1:23:05::","synset_id":"lebanese_pound.n.01"},{"index":13,"sense_key":"pound%1:23:02::","synset_id":"cypriot_pound.n.01"},{"index":14,"sense_key":"pound%1:04:00::","synset_id":"hammer.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"pound%1:23:01::","call_ids":["semeval2013.d012.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"pound%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":844,"cached_input_tokens":0,"output_tokens":155,"reasoning_output_tokens":140},"cost":{"currency":"USD","total_usd":0.00887,"input_uncached_usd":0.004220000000000001,"input_cached_usd":0.0,"output_usd":0.0046500000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.126641875016503}
{"item_id":"semeval2013.d012.s001.t005","gold_sense_keys":["usa%1:15:00::"],"candidates":[{"index":1,"sense_key":"usa%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"usa%1:14:00::","synset_id":"united_states_army.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"usa%1:15:00::","call_ids":["semeval2013.d012.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"usa%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":373,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.003215,"input_uncached_usd":0.0018650000000000001,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5145433750003576}
{"item_id":"semeval2013.d012.s002.t000","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"problem%1:26:00::","call_ids":["semeval2013.d012.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"problem%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017775,"input_uncached_usd":0.001935,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.081041333964095}
{"item_id":"semeval2013.d012.s002.t001","gold_sense_keys":["home%1:15:02::"],"candidates":[{"index":1,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":2,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":3,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"},{"index":4,"sense_key":"home%1:06:01::","synset_id":"home.n.09"},{"index":5,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":6,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"},{"index":7,"sense_key":"home%1:15:02::","synset_id":"home.n.03"},{"index":8,"sense_key":"home%1:15:03::","synset_id":"home.n.06"},{"index":9,"sense_key":"home%1:14:00::","synset_id":"family.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"home%1:15:02::","call_ids":["semeval2013.d012.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"home%1:15:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":678,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":90},"cost":{"currency":"USD","total_usd":0.00657,"input_uncached_usd":0.0033900000000000002,"input_cached_usd":0.0,"output_usd":0.00318,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.914629834005609}
{"item_id":"semeval2013.d012.s002.t002","gold_sense_keys":["industry%1:14:00::"],"candidates":[{"index":1,"sense_key":"industry%1:14:00::","synset_id":"industry.n.01"},{"index":2,"sense_key":"industry%1:07:00::","synset_id":"diligence.n.02"},{"index":3,"sense_key":"industry%1:04:00::","synset_id":"industry.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"industry%1:14:00::","call_ids":["semeval2013.d012.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"industry%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":406,"cached_input_tokens":0,"output_tokens":185,"reasoning_output_tokens":169},"cost":{"currency":"USD","total_usd":0.00758,"input_uncached_usd":0.00203,"input_cached_usd":0.0,"output_usd":0.00555,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.100325959036127}
{"item_id":"semeval2013.d012.s002.t003","gold_sense_keys":["giant%1:14:00::"],"candidates":[{"index":1,"sense_key":"giant%1:14:00::","synset_id":"giant.n.03"},{"index":2,"sense_key":"giant%1:18:01::","synset_id":"colossus.n.02"},{"index":3,"sense_key":"giant%1:05:00::","synset_id":"giant.n.01"},{"index":4,"sense_key":"giant%1:18:00::","synset_id":"giant.n.05"},{"index":5,"sense_key":"giant%1:17:00::","synset_id":"giant_star.n.01"},{"index":6,"sense_key":"giant%1:18:03::","synset_id":"giant.n.04"},{"index":7,"sense_key":"giant%1:18:02::","synset_id":"giant.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"giant%1:14:00::","call_ids":["semeval2013.d012.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"giant%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.0047350000000000005,"input_uncached_usd":0.0025450000000000004,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.597927042050287}
{"item_id":"semeval2013.d012.s002.t005","gold_sense_keys":["usa%1:15:00::"],"candidates":[{"index":1,"sense_key":"usa%1:14:00::","synset_id":"united_states_army.n.01"},{"index":2,"sense_key":"usa%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"usa%1:15:00::","call_ids":["semeval2013.d012.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"usa%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":384,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.0034200000000000003,"input_uncached_usd":0.0019200000000000003,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.788118541939184}
{"item_id":"semeval2013.d012.s002.t006","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"},{"index":2,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":3,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"},{"index":4,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d012.s002.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.0038900000000000002,"input_uncached_usd":0.0020900000000000003,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3734332909807563}
{"item_id":"semeval2013.d012.s002.t008","gold_sense_keys":["dispute%1:10:00::"],"candidates":[{"index":1,"sense_key":"dispute%1:10:00::","synset_id":"dispute.n.01"},{"index":2,"sense_key":"dispute%1:04:00::","synset_id":"dispute.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dispute%1:10:00::","call_ids":["semeval2013.d012.s002.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dispute%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":347,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.004405,"input_uncached_usd":0.0017350000000000002,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8489750410662964}
{"item_id":"semeval2013.d012.s002.t009","gold_sense_keys":["bankruptcy%1:26:00::"],"candidates":[{"index":1,"sense_key":"bankruptcy%1:26:01::","synset_id":"bankruptcy.n.01"},{"index":2,"sense_key":"bankruptcy%1:04:00::","synset_id":"bankruptcy.n.03"},{"index":3,"sense_key":"bankruptcy%1:26:00::","synset_id":"bankruptcy.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bankruptcy%1:26:00::","call_ids":["semeval2013.d012.s002.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bankruptcy%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":203,"reasoning_output_tokens":187},"cost":{"currency":"USD","total_usd":0.008025000000000001,"input_uncached_usd":0.001935,"input_cached_usd":0.0,"output_usd":0.00609,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.991169125074521}
{"item_id":"semeval2013.d012.s003.t000","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":4,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":5,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":6,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":7,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":8,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":9,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":10,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"},{"index":11,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"issue%1:09:01::","call_ids":["semeval2013.d012.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":845,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":126},"cost":{"currency":"USD","total_usd":0.008485,"input_uncached_usd":0.0042250000000000005,"input_cached_usd":0.0,"output_usd":0.00426,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.706530124996789}
{"item_id":"semeval2013.d012.s003.t001","gold_sense_keys":["sale%1:04:02::"],"candidates":[{"index":1,"sense_key":"sale%1:26:00::","synset_id":"sale.n.04"},{"index":2,"sense_key":"sale%1:04:00::","synset_id":"sale.n.01"},{"index":3,"sense_key":"sale%1:10:00::","synset_id":"sale.n.05"},{"index":4,"sense_key":"sale%1:04:02::","synset_id":"sale.n.02"},{"index":5,"sense_key":"sale%1:04:01::","synset_id":"sale.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"sale%1:04:02::","call_ids":["semeval2013.d012.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"sale%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":571,"cached_input_tokens":0,"output_tokens":274,"reasoning_output_tokens":258},"cost":{"currency":"USD","total_usd":0.011075,"input_uncached_usd":0.0028550000000000003,"input_cached_usd":0.0,"output_usd":0.00822,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.419184749945998}
{"item_id":"semeval2013.d012.s004.t000","gold_sense_keys":["regulator%1:18:00::"],"candidates":[{"index":1,"sense_key":"regulator%1:06:00::","synset_id":"regulator.n.01"},{"index":2,"sense_key":"regulator%1:06:01::","synset_id":"governor.n.02"},{"index":3,"sense_key":"regulator%1:18:00::","synset_id":"regulator.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"regulator%1:18:00::","call_ids":["semeval2013.d012.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"regulator%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.0036,"input_uncached_usd":0.00216,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5813927920535207}
{"item_id":"semeval2013.d012.s004.t001","gold_sense_keys":["string%1:14:00::"],"candidates":[{"index":1,"sense_key":"string%1:06:01::","synset_id":"drawstring.n.01"},{"index":2,"sense_key":"string%1:10:00::","synset_id":"string.n.05"},{"index":3,"sense_key":"string%1:06:03::","synset_id":"string.n.03"},{"index":4,"sense_key":"string%1:06:02::","synset_id":"chain.n.10"},{"index":5,"sense_key":"string%1:14:00::","synset_id":"string.n.04"},{"index":6,"sense_key":"string%1:27:00::","synset_id":"string.n.07"},{"index":7,"sense_key":"string%1:06:00::","synset_id":"string.n.01"},{"index":8,"sense_key":"string%1:06:05::","synset_id":"bowed_stringed_instrument.n.01"},{"index":9,"sense_key":"string%1:06:04::","synset_id":"string.n.09"},{"index":10,"sense_key":"string%1:17:00::","synset_id":"string.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"string%1:14:00::","call_ids":["semeval2013.d012.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"string%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":708,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.007050000000000001,"input_uncached_usd":0.00354,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.429142332985066}
{"item_id":"semeval2013.d012.s004.t002","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":2,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":3,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":4,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":5,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"},{"index":6,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":7,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":8,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":9,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":10,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d012.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":764,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.00613,"input_uncached_usd":0.0038200000000000005,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.099544540978968}
{"item_id":"semeval2013.d012.s004.t003","gold_sense_keys":["promise%1:10:00::"],"candidates":[{"index":1,"sense_key":"promise%1:09:00::","synset_id":"promise.n.02"},{"index":2,"sense_key":"promise%1:10:00::","synset_id":"promise.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"promise%1:10:00::","call_ids":["semeval2013.d012.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"promise%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":399,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.004605,"input_uncached_usd":0.0019950000000000002,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5140348330605775}
{"item_id":"semeval2013.d012.s005.t000","gold_sense_keys":["value%1:21:00::"],"candidates":[{"index":1,"sense_key":"value%1:28:00::","synset_id":"value.n.05"},{"index":2,"sense_key":"value%1:21:00::","synset_id":"value.n.03"},{"index":3,"sense_key":"value%1:07:01::","synset_id":"value.n.04"},{"index":4,"sense_key":"value%1:09:01::","synset_id":"value.n.06"},{"index":5,"sense_key":"value%1:09:00::","synset_id":"value.n.01"},{"index":6,"sense_key":"value%1:07:00::","synset_id":"value.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"value%1:21:00::","call_ids":["semeval2013.d012.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"value%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.00542,"input_uncached_usd":0.00299,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.382347790990025}
{"item_id":"semeval2013.d012.s005.t001","gold_sense_keys":["security%1:21:04::"],"candidates":[{"index":1,"sense_key":"security%1:21:00::","synset_id":"security.n.05"},{"index":2,"sense_key":"security%1:21:04::","synset_id":"security.n.04"},{"index":3,"sense_key":"security%1:14:00::","synset_id":"security.n.06"},{"index":4,"sense_key":"security%1:26:00::","synset_id":"security.n.01"},{"index":5,"sense_key":"security%1:06:00::","synset_id":"security_system.n.02"},{"index":6,"sense_key":"security%1:21:01::","synset_id":"security.n.02"},{"index":7,"sense_key":"security%1:12:00::","synset_id":"security.n.03"},{"index":8,"sense_key":"security%1:10:00::","synset_id":"security.n.07"},{"index":9,"sense_key":"security%1:04:00::","synset_id":"security.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"security%1:21:04::","call_ids":["semeval2013.d012.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"security%1:21:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.0052450000000000005,"input_uncached_usd":0.0036850000000000003,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.678971416899003}
{"item_id":"semeval2013.d012.s005.t002","gold_sense_keys":["crisis%1:26:00::"],"candidates":[{"index":1,"sense_key":"crisis%1:11:00::","synset_id":"crisis.n.02"},{"index":2,"sense_key":"crisis%1:26:00::","synset_id":"crisis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"crisis%1:26:00::","call_ids":["semeval2013.d012.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"crisis%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":423,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.003705,"input_uncached_usd":0.002115,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5270031250547618}
{"item_id":"semeval2013.d012.s006.t000","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":4,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":5,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":6,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":7,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"},{"index":8,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":9,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":10,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":11,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"issue%1:09:01::","call_ids":["semeval2013.d012.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":888,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":81},"cost":{"currency":"USD","total_usd":0.00732,"input_uncached_usd":0.00444,"input_cached_usd":0.0,"output_usd":0.00288,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.891156541998498}
{"item_id":"semeval2013.d012.s006.t004","gold_sense_keys":["new_york%1:15:00::","new_york%1:15:01::"],"candidates":[{"index":1,"sense_key":"new_york%1:15:00::","synset_id":"new_york.n.02"},{"index":2,"sense_key":"new_york%1:15:01::","synset_id":"new_york.n.01"},{"index":3,"sense_key":"new_york%1:15:02::","synset_id":"new_york.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"new_york%1:15:01::","call_ids":["semeval2013.d012.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"new_york%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00421,"input_uncached_usd":0.00238,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4706998330075294}
{"item_id":"semeval2013.d012.s007.t000","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":3,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"},{"index":4,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":5,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":6,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":7,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":8,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":9,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":10,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d012.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00558,"input_uncached_usd":0.00387,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.036326083005406}
{"item_id":"semeval2013.d012.s007.t002","gold_sense_keys":["settlement%1:10:00::"],"candidates":[{"index":1,"sense_key":"settlement%1:04:00::","synset_id":"liquidation.n.01"},{"index":2,"sense_key":"settlement%1:09:00::","synset_id":"settlement.n.05"},{"index":3,"sense_key":"settlement%1:14:01::","synset_id":"village.n.01"},{"index":4,"sense_key":"settlement%1:10:00::","synset_id":"settlement.n.03"},{"index":5,"sense_key":"settlement%1:15:00::","synset_id":"settlement.n.06"},{"index":6,"sense_key":"settlement%1:14:00::","synset_id":"colony.n.01"},{"index":7,"sense_key":"settlement%1:04:01::","synset_id":"colonization.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"settlement%1:09:00::","call_ids":["semeval2013.d012.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"settlement%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018935,"input_uncached_usd":0.003095,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.229978582938202}
{"item_id":"semeval2013.d012.s008.t000","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":2,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"},{"index":3,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"},{"index":4,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d012.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":438,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.00408,"input_uncached_usd":0.00219,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2810611670138314}
{"item_id":"semeval2013.d012.s009.t001","gold_sense_keys":["responsiveness%1:07:00::"],"candidates":[{"index":1,"sense_key":"responsiveness%1:07:00::","synset_id":"responsiveness.n.02"},{"index":2,"sense_key":"responsiveness%1:09:00::","synset_id":"responsiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"responsiveness%1:07:00::","call_ids":["semeval2013.d012.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"responsiveness%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":372,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.0045000000000000005,"input_uncached_usd":0.00186,"input_cached_usd":0.0,"output_usd":0.00264,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0994439169298857}
{"item_id":"semeval2013.d012.s009.t002","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":2,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":3,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":4,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":5,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"},{"index":6,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":7,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":8,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":9,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":10,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d012.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":746,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.0056500000000000005,"input_uncached_usd":0.0037300000000000002,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.375931750051677}
{"item_id":"semeval2013.d012.s010.t002","gold_sense_keys":["bankruptcy%1:04:00::"],"candidates":[{"index":1,"sense_key":"bankruptcy%1:04:00::","synset_id":"bankruptcy.n.03"},{"index":2,"sense_key":"bankruptcy%1:26:00::","synset_id":"bankruptcy.n.02"},{"index":3,"sense_key":"bankruptcy%1:26:01::","synset_id":"bankruptcy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bankruptcy%1:26:00::","call_ids":["semeval2013.d012.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bankruptcy%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":405,"cached_input_tokens":0,"output_tokens":404,"reasoning_output_tokens":388},"cost":{"currency":"USD","total_usd":0.014145000000000001,"input_uncached_usd":0.0020250000000000003,"input_cached_usd":0.0,"output_usd":0.01212,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.726714833057486}
{"item_id":"semeval2013.d012.s010.t004","gold_sense_keys":["deposit%1:21:01::"],"candidates":[{"index":1,"sense_key":"deposit%1:10:01::","synset_id":"deposit.n.07"},{"index":2,"sense_key":"deposit%1:19:00::","synset_id":"deposit.n.01"},{"index":3,"sense_key":"deposit%1:22:00::","synset_id":"deposition.n.01"},{"index":4,"sense_key":"deposit%1:21:02::","synset_id":"deposit.n.06"},{"index":5,"sense_key":"deposit%1:21:01::","synset_id":"deposit.n.04"},{"index":6,"sense_key":"deposit%1:21:00::","synset_id":"down_payment.n.01"},{"index":7,"sense_key":"deposit%1:06:00::","synset_id":"depository.n.01"},{"index":8,"sense_key":"deposit%1:17:00::","synset_id":"sediment.n.01"},{"index":9,"sense_key":"deposit%1:04:00::","synset_id":"deposit.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"deposit%1:21:01::","call_ids":["semeval2013.d012.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"deposit%1:21:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.00477,"input_uncached_usd":0.0029100000000000003,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.189315958996303}
{"item_id":"semeval2013.d012.s011.t000","gold_sense_keys":["compensation%1:21:00::"],"candidates":[{"index":1,"sense_key":"compensation%1:04:00::","synset_id":"recompense.n.02"},{"index":2,"sense_key":"compensation%1:21:00::","synset_id":"compensation.n.01"},{"index":3,"sense_key":"compensation%1:22:00::","synset_id":"compensation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"compensation%1:21:00::","call_ids":["semeval2013.d012.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"compensation%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":386,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.00484,"input_uncached_usd":0.00193,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.40608829096891}
{"item_id":"semeval2013.d012.s011.t002","gold_sense_keys":["billion%1:23:00::"],"candidates":[{"index":1,"sense_key":"billion%1:23:01::","synset_id":"billion.n.01"},{"index":2,"sense_key":"billion%1:23:00::","synset_id":"billion.n.03"},{"index":3,"sense_key":"billion%1:23:02::","synset_id":"million.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"billion%1:23:00::","call_ids":["semeval2013.d012.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"billion%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":126},"cost":{"currency":"USD","total_usd":0.006370000000000001,"input_uncached_usd":0.0021100000000000003,"input_cached_usd":0.0,"output_usd":0.00426,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0760499159805477}
{"item_id":"semeval2013.d012.s012.t000","gold_sense_keys":["settlement%1:10:00::"],"candidates":[{"index":1,"sense_key":"settlement%1:14:01::","synset_id":"village.n.01"},{"index":2,"sense_key":"settlement%1:10:00::","synset_id":"settlement.n.03"},{"index":3,"sense_key":"settlement%1:04:01::","synset_id":"colonization.n.01"},{"index":4,"sense_key":"settlement%1:15:00::","synset_id":"settlement.n.06"},{"index":5,"sense_key":"settlement%1:04:00::","synset_id":"liquidation.n.01"},{"index":6,"sense_key":"settlement%1:14:00::","synset_id":"colony.n.01"},{"index":7,"sense_key":"settlement%1:09:00::","synset_id":"settlement.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"settlement%1:10:00::","call_ids":["semeval2013.d012.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"settlement%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":1564,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.049745000000000004,"input_uncached_usd":0.0028250000000000003,"input_cached_usd":0.0,"output_usd":0.04692,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":36.06141233397648}
{"item_id":"semeval2013.d012.s013.t000","gold_sense_keys":["regulator%1:18:00::"],"candidates":[{"index":1,"sense_key":"regulator%1:06:01::","synset_id":"governor.n.02"},{"index":2,"sense_key":"regulator%1:06:00::","synset_id":"regulator.n.01"},{"index":3,"sense_key":"regulator%1:18:00::","synset_id":"regulator.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"regulator%1:18:00::","call_ids":["semeval2013.d012.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"regulator%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":388,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.00356,"input_uncached_usd":0.00194,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8553765419637784}
{"item_id":"semeval2013.d012.s013.t001","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"},{"index":2,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":3,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":4,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":5,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":6,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":7,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":8,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":9,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":10,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d012.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":719,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.005335,"input_uncached_usd":0.003595,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8738331249915063}
{"item_id":"semeval2013.d012.s013.t002","gold_sense_keys":["summer%1:28:00::"],"candidates":[{"index":1,"sense_key":"summer%1:28:00::","synset_id":"summer.n.01"},{"index":2,"sense_key":"summer%1:28:01::","synset_id":"summer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"summer%1:28:00::","call_ids":["semeval2013.d012.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"summer%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":368,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.00334,"input_uncached_usd":0.00184,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4561548749916255}
{"item_id":"semeval2013.d012.s016.t000","gold_sense_keys":["fate%1:11:00::"],"candidates":[{"index":1,"sense_key":"fate%1:26:00::","synset_id":"fortune.n.04"},{"index":2,"sense_key":"fate%1:11:00::","synset_id":"destiny.n.01"},{"index":3,"sense_key":"fate%1:18:00::","synset_id":"destiny.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fate%1:11:00::","call_ids":["semeval2013.d012.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fate%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":491,"reasoning_output_tokens":475},"cost":{"currency":"USD","total_usd":0.016815,"input_uncached_usd":0.002085,"input_cached_usd":0.0,"output_usd":0.01473,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.508755959104747}
{"item_id":"semeval2013.d012.s016.t001","gold_sense_keys":["number%1:07:00::"],"candidates":[{"index":1,"sense_key":"number%1:14:00::","synset_id":"number.n.07"},{"index":2,"sense_key":"number%1:10:04::","synset_id":"act.n.04"},{"index":3,"sense_key":"number%1:23:00::","synset_id":"number.n.02"},{"index":4,"sense_key":"number%1:07:01::","synset_id":"number.n.09"},{"index":5,"sense_key":"number%1:07:00::","synset_id":"number.n.01"},{"index":6,"sense_key":"number%1:10:02::","synset_id":"number.n.08"},{"index":7,"sense_key":"number%1:06:00::","synset_id":"number.n.11"},{"index":8,"sense_key":"number%1:10:00::","synset_id":"issue.n.02"},{"index":9,"sense_key":"number%1:10:01::","synset_id":"numeral.n.01"},{"index":10,"sense_key":"number%1:10:05::","synset_id":"number.n.10"},{"index":11,"sense_key":"number%1:10:03::","synset_id":"phone_number.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"number%1:07:00::","call_ids":["semeval2013.d012.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"number%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":751,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.006215,"input_uncached_usd":0.003755,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0796358750667423}
{"item_id":"semeval2013.d012.s016.t002","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":2,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":3,"sense_key":"company%1:14:05::","synset_id":"company.n.09"},{"index":4,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":5,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":6,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":7,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":8,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":9,"sense_key":"company%1:14:02::","synset_id":"company.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d012.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.00468,"input_uncached_usd":0.0030900000000000003,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7832162500126287}
{"item_id":"semeval2013.d012.s017.t002","gold_sense_keys":["height%1:26:00::"],"candidates":[{"index":1,"sense_key":"height%1:26:00::","synset_id":"acme.n.01"},{"index":2,"sense_key":"height%1:07:00::","synset_id":"height.n.01"},{"index":3,"sense_key":"height%1:07:01::","synset_id":"stature.n.02"},{"index":4,"sense_key":"height%1:07:02::","synset_id":"altitude.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"height%1:26:00::","call_ids":["semeval2013.d012.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"height%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.00405,"input_uncached_usd":0.00222,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6361046669771895}
{"item_id":"semeval2013.d012.s017.t003","gold_sense_keys":["crisis%1:26:00::"],"candidates":[{"index":1,"sense_key":"crisis%1:11:00::","synset_id":"crisis.n.02"},{"index":2,"sense_key":"crisis%1:26:00::","synset_id":"crisis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"crisis%1:26:00::","call_ids":["semeval2013.d012.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"crisis%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":341,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.004015,"input_uncached_usd":0.0017050000000000001,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1525837909430265}
{"item_id":"semeval2013.d012.s018.t000","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":2,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":3,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":6,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":9,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"},{"index":10,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d012.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":715,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.005825,"input_uncached_usd":0.0035750000000000005,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7533071669749916}
{"item_id":"semeval2013.d012.s018.t001","gold_sense_keys":["wind%1:19:01::"],"candidates":[{"index":1,"sense_key":"wind%1:10:00::","synset_id":"tip.n.03"},{"index":2,"sense_key":"wind%1:06:00::","synset_id":"wind_instrument.n.01"},{"index":3,"sense_key":"wind%1:19:01::","synset_id":"wind.n.02"},{"index":4,"sense_key":"wind%1:19:00::","synset_id":"wind.n.01"},{"index":5,"sense_key":"wind%1:04:01::","synset_id":"wind.n.03"},{"index":6,"sense_key":"wind%1:04:00::","synset_id":"wind.n.08"},{"index":7,"sense_key":"wind%1:04:02::","synset_id":"fart.n.01"},{"index":8,"sense_key":"wind%1:10:01::","synset_id":"wind.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"wind%1:19:01::","call_ids":["semeval2013.d012.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"wind%1:19:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":72},"cost":{"currency":"USD","total_usd":0.00578,"input_uncached_usd":0.00317,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.780302375089377}
{"item_id":"semeval2013.d012.s018.t002","gold_sense_keys":["usa%1:15:00::"],"candidates":[{"index":1,"sense_key":"usa%1:14:00::","synset_id":"united_states_army.n.01"},{"index":2,"sense_key":"usa%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"usa%1:15:00::","call_ids":["semeval2013.d012.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"usa%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":39,"reasoning_output_tokens":24},"cost":{"currency":"USD","total_usd":0.0031200000000000004,"input_uncached_usd":0.0019500000000000001,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.555258500040509}
{"item_id":"semeval2013.d012.s018.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:14:00::","synset_id":"class.n.06"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":4,"sense_key":"year%1:28:00::","synset_id":"year.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d012.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004315,"input_uncached_usd":0.0023350000000000003,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.73799570905976}
{"item_id":"semeval2013.d012.s019.t001","gold_sense_keys":["regulator%1:18:00::"],"candidates":[{"index":1,"sense_key":"regulator%1:06:01::","synset_id":"governor.n.02"},{"index":2,"sense_key":"regulator%1:06:00::","synset_id":"regulator.n.01"},{"index":3,"sense_key":"regulator%1:18:00::","synset_id":"regulator.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"regulator%1:18:00::","call_ids":["semeval2013.d012.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"regulator%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":376,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.0034100000000000003,"input_uncached_usd":0.0018800000000000002,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5056935829343274}
{"item_id":"semeval2013.d012.s019.t002","gold_sense_keys":["number%1:07:00::"],"candidates":[{"index":1,"sense_key":"number%1:06:00::","synset_id":"number.n.11"},{"index":2,"sense_key":"number%1:10:01::","synset_id":"numeral.n.01"},{"index":3,"sense_key":"number%1:23:00::","synset_id":"number.n.02"},{"index":4,"sense_key":"number%1:10:03::","synset_id":"phone_number.n.01"},{"index":5,"sense_key":"number%1:10:04::","synset_id":"act.n.04"},{"index":6,"sense_key":"number%1:07:01::","synset_id":"number.n.09"},{"index":7,"sense_key":"number%1:07:00::","synset_id":"number.n.01"},{"index":8,"sense_key":"number%1:10:02::","synset_id":"number.n.08"},{"index":9,"sense_key":"number%1:10:00::","synset_id":"issue.n.02"},{"index":10,"sense_key":"number%1:10:05::","synset_id":"number.n.10"},{"index":11,"sense_key":"number%1:14:00::","synset_id":"number.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"number%1:07:00::","call_ids":["semeval2013.d012.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"number%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":748,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.00593,"input_uncached_usd":0.0037400000000000003,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7912055830238387}
{"item_id":"semeval2013.d012.s019.t003","gold_sense_keys":["claim%1:04:00::","claim%1:10:00::"],"candidates":[{"index":1,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"},{"index":2,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":3,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":4,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":5,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":6,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"claim%1:10:00::","call_ids":["semeval2013.d012.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"claim%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":397,"reasoning_output_tokens":381},"cost":{"currency":"USD","total_usd":0.014410000000000001,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.01191,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.395981332985684}
{"item_id":"semeval2013.d012.s019.t004","gold_sense_keys":["compensation%1:21:00::"],"candidates":[{"index":1,"sense_key":"compensation%1:21:00::","synset_id":"compensation.n.01"},{"index":2,"sense_key":"compensation%1:22:00::","synset_id":"compensation.n.02"},{"index":3,"sense_key":"compensation%1:04:00::","synset_id":"recompense.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compensation%1:21:00::","call_ids":["semeval2013.d012.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compensation%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":376,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.00503,"input_uncached_usd":0.0018800000000000002,"input_cached_usd":0.0,"output_usd":0.00315,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4117434580111876}
{"item_id":"semeval2013.d012.s019.t005","gold_sense_keys":["misconduct%1:04:00::"],"candidates":[{"index":1,"sense_key":"misconduct%1:04:00::","synset_id":"wrongdoing.n.02"},{"index":2,"sense_key":"misconduct%1:04:01::","synset_id":"misconduct.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"misconduct%1:04:00::","call_ids":["semeval2013.d012.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"misconduct%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":341,"cached_input_tokens":0,"output_tokens":373,"reasoning_output_tokens":357},"cost":{"currency":"USD","total_usd":0.012895,"input_uncached_usd":0.0017050000000000001,"input_cached_usd":0.0,"output_usd":0.01119,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.455428667017259}
{"item_id":"semeval2013.d012.s020.t000","gold_sense_keys":["wave%1:04:02::"],"candidates":[{"index":1,"sense_key":"wave%1:04:02::","synset_id":"wave.n.02"},{"index":2,"sense_key":"wave%1:18:00::","synset_id":"wave.n.09"},{"index":3,"sense_key":"wave%1:25:00::","synset_id":"wave.n.07"},{"index":4,"sense_key":"wave%1:19:00::","synset_id":"wave.n.08"},{"index":5,"sense_key":"wave%1:11:01::","synset_id":"wave.n.01"},{"index":6,"sense_key":"wave%1:08:00::","synset_id":"wave.n.06"},{"index":7,"sense_key":"wave%1:11:02::","synset_id":"wave.n.04"},{"index":8,"sense_key":"wave%1:11:00::","synset_id":"wave.n.03"},{"index":9,"sense_key":"wave%1:10:01::","synset_id":"wave.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wave%1:04:02::","call_ids":["semeval2013.d012.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wave%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018875,"input_uncached_usd":0.0030350000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.858565957983956}
{"item_id":"semeval2013.d012.s020.t001","gold_sense_keys":["claim%1:04:00::","claim%1:10:00::"],"candidates":[{"index":1,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":2,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":3,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":4,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"},{"index":5,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":6,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"claim%1:10:00::","call_ids":["semeval2013.d012.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"claim%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":259,"reasoning_output_tokens":244},"cost":{"currency":"USD","total_usd":0.010365,"input_uncached_usd":0.002595,"input_cached_usd":0.0,"output_usd":0.00777,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.408893707906827}
{"item_id":"semeval2013.d012.s021.t000","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":2,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":3,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"},{"index":4,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":7,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":8,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":9,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":10,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d012.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":751,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.005315,"input_uncached_usd":0.003755,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6181911249877885}
{"item_id":"semeval2013.d012.s021.t007","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"},{"index":2,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":3,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"},{"index":4,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d012.s021.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.0042450000000000005,"input_uncached_usd":0.002295,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.992039707954973}
{"item_id":"semeval2013.d012.s022.t002","gold_sense_keys":["value%1:21:00::"],"candidates":[{"index":1,"sense_key":"value%1:07:01::","synset_id":"value.n.04"},{"index":2,"sense_key":"value%1:09:01::","synset_id":"value.n.06"},{"index":3,"sense_key":"value%1:21:00::","synset_id":"value.n.03"},{"index":4,"sense_key":"value%1:09:00::","synset_id":"value.n.01"},{"index":5,"sense_key":"value%1:28:00::","synset_id":"value.n.05"},{"index":6,"sense_key":"value%1:07:00::","synset_id":"value.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"value%1:21:00::","call_ids":["semeval2013.d012.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"value%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.0049250000000000006,"input_uncached_usd":0.0028250000000000003,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.660420542000793}
{"item_id":"semeval2013.d012.s022.t003","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"},{"index":2,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":3,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"},{"index":4,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d012.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.004275,"input_uncached_usd":0.0023250000000000002,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4732667499920353}
{"item_id":"semeval2013.d012.s023.t000","gold_sense_keys":["regulator%1:18:00::"],"candidates":[{"index":1,"sense_key":"regulator%1:18:00::","synset_id":"regulator.n.02"},{"index":2,"sense_key":"regulator%1:06:01::","synset_id":"governor.n.02"},{"index":3,"sense_key":"regulator%1:06:00::","synset_id":"regulator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"regulator%1:18:00::","call_ids":["semeval2013.d012.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"regulator%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.0034500000000000004,"input_uncached_usd":0.0021000000000000003,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4900980830425397}
{"item_id":"semeval2013.d012.s023.t001","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":2,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":5,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":8,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":9,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":10,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d012.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":751,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.005465,"input_uncached_usd":0.003755,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.343659917009063}
{"item_id":"semeval2013.d012.s023.t002","gold_sense_keys":["responsibility%1:07:00::"],"candidates":[{"index":1,"sense_key":"responsibility%1:07:00::","synset_id":"responsibility.n.03"},{"index":2,"sense_key":"responsibility%1:04:00::","synset_id":"duty.n.01"},{"index":3,"sense_key":"responsibility%1:26:00::","synset_id":"province.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"responsibility%1:07:00::","call_ids":["semeval2013.d012.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"responsibility%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018265,"input_uncached_usd":0.002425,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.215969874989241}
{"item_id":"semeval2013.d012.s023.t004","gold_sense_keys":["sum%1:21:00::"],"candidates":[{"index":1,"sense_key":"sum%1:09:00::","synset_id":"kernel.n.03"},{"index":2,"sense_key":"sum%1:09:01::","synset_id":"sum.n.02"},{"index":3,"sense_key":"sum%1:14:01::","synset_id":"union.n.08"},{"index":4,"sense_key":"sum%1:14:00::","synset_id":"sum.n.03"},{"index":5,"sense_key":"sum%1:06:00::","synset_id":"sum.n.05"},{"index":6,"sense_key":"sum%1:21:00::","synset_id":"sum.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"sum%1:21:00::","call_ids":["semeval2013.d012.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"sum%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":96},"cost":{"currency":"USD","total_usd":0.00621,"input_uncached_usd":0.00285,"input_cached_usd":0.0,"output_usd":0.00336,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7609442089451477}
{"item_id":"semeval2013.d012.s024.t000","gold_sense_keys":["claim%1:04:00::","claim%1:10:00::"],"candidates":[{"index":1,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":2,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":3,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"},{"index":4,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":5,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":6,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"claim%1:10:00::","call_ids":["semeval2013.d012.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"claim%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.034034999999999996,"input_uncached_usd":0.0026550000000000002,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":16.914762999978848}
{"item_id":"semeval2013.d012.s024.t001","gold_sense_keys":["intention%1:09:00::"],"candidates":[{"index":1,"sense_key":"intention%1:04:00::","synset_id":"intention.n.03"},{"index":2,"sense_key":"intention%1:09:01::","synset_id":"intention.n.02"},{"index":3,"sense_key":"intention%1:09:00::","synset_id":"purpose.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"intention%1:09:00::","call_ids":["semeval2013.d012.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"intention%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":1725,"reasoning_output_tokens":1692},"cost":{"currency":"USD","total_usd":0.053884999999999995,"input_uncached_usd":0.002135,"input_cached_usd":0.0,"output_usd":0.051750000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":31.748891082941554}
{"item_id":"semeval2013.d012.s024.t002","gold_sense_keys":["claim%1:04:00::","claim%1:10:00::"],"candidates":[{"index":1,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"},{"index":2,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":3,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":4,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":5,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":6,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"claim%1:10:00::","call_ids":["semeval2013.d012.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"claim%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018495,"input_uncached_usd":0.0026550000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.548898875014856}
{"item_id":"semeval2015.d000.s000.t000","gold_sense_keys":["document%1:10:00::"],"candidates":[{"index":1,"sense_key":"document%1:10:00::","synset_id":"document.n.01"},{"index":2,"sense_key":"document%1:06:00::","synset_id":"document.n.02"},{"index":3,"sense_key":"document%1:10:01::","synset_id":"text_file.n.01"},{"index":4,"sense_key":"document%1:21:00::","synset_id":"document.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"document%1:10:00::","call_ids":["semeval2015.d000.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"document%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":322,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.00434,"input_uncached_usd":0.00161,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9226995840435848}
{"item_id":"semeval2015.d000.s000.t003","gold_sense_keys":["public%3:00:00::"],"candidates":[{"index":1,"sense_key":"public%3:00:00::","synset_id":"public.a.01"},{"index":2,"sense_key":"public%5:00:00:common:02","synset_id":"public.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"public%3:00:00::","call_ids":["semeval2015.d000.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"public%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":271,"cached_input_tokens":0,"output_tokens":141,"reasoning_output_tokens":126},"cost":{"currency":"USD","total_usd":0.0055850000000000006,"input_uncached_usd":0.001355,"input_cached_usd":0.0,"output_usd":0.00423,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.931223207968287}
{"item_id":"semeval2015.d000.s000.t004","gold_sense_keys":["assessment%1:04:00::"],"candidates":[{"index":1,"sense_key":"assessment%1:09:00::","synset_id":"appraisal.n.01"},{"index":2,"sense_key":"assessment%1:07:00::","synset_id":"assessment.n.03"},{"index":3,"sense_key":"assessment%1:21:00::","synset_id":"assessment.n.02"},{"index":4,"sense_key":"assessment%1:04:00::","synset_id":"judgment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"assessment%1:04:00::","call_ids":["semeval2015.d000.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"assessment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":332,"cached_input_tokens":0,"output_tokens":218,"reasoning_output_tokens":202},"cost":{"currency":"USD","total_usd":0.0082,"input_uncached_usd":0.0016600000000000002,"input_cached_usd":0.0,"output_usd":0.00654,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.298560999915935}
{"item_id":"semeval2015.d000.s000.t005","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":2,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":3,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"},{"index":4,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":5,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":6,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":7,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2015.d000.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.00506,"input_uncached_usd":0.0026000000000000003,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6194174169795588}
{"item_id":"semeval2015.d000.s001.t000","gold_sense_keys":["explain%2:32:00::","explain%2:32:01::"],"candidates":[{"index":1,"sense_key":"explain%2:32:00::","synset_id":"explain.v.01"},{"index":2,"sense_key":"explain%2:32:01::","synset_id":"explain.v.02"},{"index":3,"sense_key":"explain%2:32:02::","synset_id":"excuse.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"explain%2:32:00::","call_ids":["semeval2015.d000.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"explain%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":363,"cached_input_tokens":0,"output_tokens":395,"reasoning_output_tokens":380},"cost":{"currency":"USD","total_usd":0.013665000000000002,"input_uncached_usd":0.0018150000000000002,"input_cached_usd":0.0,"output_usd":0.011850000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.371599165955558}
{"item_id":"semeval2015.d000.s001.t001","gold_sense_keys":["committee%1:14:00::"],"candidates":[{"index":1,"sense_key":"committee%1:14:00::","synset_id":"committee.n.01"},{"index":2,"sense_key":"committee%1:14:01::","synset_id":"committee.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"committee%1:14:00::","call_ids":["semeval2015.d000.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"committee%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":306,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.00378,"input_uncached_usd":0.0015300000000000001,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.445446499972604}
{"item_id":"semeval2015.d000.s001.t003","gold_sense_keys":["product%1:06:01::"],"candidates":[{"index":1,"sense_key":"product%1:14:00::","synset_id":"intersection.n.04"},{"index":2,"sense_key":"product%1:19:00::","synset_id":"product.n.05"},{"index":3,"sense_key":"product%1:09:00::","synset_id":"product.n.03"},{"index":4,"sense_key":"product%1:27:00::","synset_id":"product.n.04"},{"index":5,"sense_key":"product%1:06:01::","synset_id":"merchandise.n.01"},{"index":6,"sense_key":"product%1:06:00::","synset_id":"product.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"product%1:06:01::","call_ids":["semeval2015.d000.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"product%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":256,"reasoning_output_tokens":240},"cost":{"currency":"USD","total_usd":0.01022,"input_uncached_usd":0.00254,"input_cached_usd":0.0,"output_usd":0.00768,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.770190584007651}
{"item_id":"semeval2015.d000.s001.t004","gold_sense_keys":["human%3:01:01::"],"candidates":[{"index":1,"sense_key":"human%3:00:00::","synset_id":"human.a.03"},{"index":2,"sense_key":"human%3:01:00::","synset_id":"human.a.01"},{"index":3,"sense_key":"human%3:01:01::","synset_id":"human.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"human%3:01:01::","call_ids":["semeval2015.d000.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"human%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":332,"cached_input_tokens":0,"output_tokens":313,"reasoning_output_tokens":297},"cost":{"currency":"USD","total_usd":0.011049999999999999,"input_uncached_usd":0.0016600000000000002,"input_cached_usd":0.0,"output_usd":0.00939,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.7645442499779165}
{"item_id":"semeval2015.d000.s001.t005","gold_sense_keys":["use%1:04:00::","use%1:07:02::"],"candidates":[{"index":1,"sense_key":"use%1:04:02::","synset_id":"manipulation.n.01"},{"index":2,"sense_key":"use%1:22:00::","synset_id":"consumption.n.03"},{"index":3,"sense_key":"use%1:07:03::","synset_id":"use.n.07"},{"index":4,"sense_key":"use%1:07:00::","synset_id":"function.n.02"},{"index":5,"sense_key":"use%1:07:02::","synset_id":"use.n.03"},{"index":6,"sense_key":"use%1:04:00::","synset_id":"use.n.01"},{"index":7,"sense_key":"use%1:04:01::","synset_id":"habit.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"use%1:04:00::","call_ids":["semeval2015.d000.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"use%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":214,"reasoning_output_tokens":198},"cost":{"currency":"USD","total_usd":0.009345000000000001,"input_uncached_usd":0.002925,"input_cached_usd":0.0,"output_usd":0.00642,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.040722667006776}
{"item_id":"semeval2015.d000.s001.t006","gold_sense_keys":["assess%2:31:00::"],"candidates":[{"index":1,"sense_key":"assess%2:40:00::","synset_id":"tax.v.02"},{"index":2,"sense_key":"assess%2:31:01::","synset_id":"assess.v.04"},{"index":3,"sense_key":"assess%2:31:00::","synset_id":"measure.v.04"},{"index":4,"sense_key":"assess%2:40:01::","synset_id":"assess.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"assess%2:31:00::","call_ids":["semeval2015.d000.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"assess%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":408,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.0033900000000000002,"input_uncached_usd":0.00204,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3530385000631213}
{"item_id":"semeval2015.d000.s001.t007","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:10:01::","synset_id":"study.n.10"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":6,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":7,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":8,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":9,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":10,"sense_key":"study%1:18:00::","synset_id":"study.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d000.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":278,"reasoning_output_tokens":263},"cost":{"currency":"USD","total_usd":0.011555000000000001,"input_uncached_usd":0.0032150000000000004,"input_cached_usd":0.0,"output_usd":0.00834,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.422541082953103}
{"item_id":"semeval2015.d000.s001.t008","gold_sense_keys":["perform%2:36:00::"],"candidates":[{"index":1,"sense_key":"perform%2:41:00::","synset_id":"do.v.03"},{"index":2,"sense_key":"perform%2:41:02::","synset_id":"perform.v.02"},{"index":3,"sense_key":"perform%2:36:01::","synset_id":"perform.v.03"},{"index":4,"sense_key":"perform%2:36:00::","synset_id":"perform.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"perform%2:36:00::","call_ids":["semeval2015.d000.s001.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"perform%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":388,"cached_input_tokens":0,"output_tokens":1139,"reasoning_output_tokens":1111},"cost":{"currency":"USD","total_usd":0.036109999999999996,"input_uncached_usd":0.00194,"input_cached_usd":0.0,"output_usd":0.03417,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":21.28352175001055}
{"item_id":"semeval2015.d000.s001.t009","gold_sense_keys":["reach%2:38:06::","reach%2:41:00::"],"candidates":[{"index":1,"sense_key":"reach%2:38:01::","synset_id":"reach.v.01"},{"index":2,"sense_key":"reach%2:32:00::","synset_id":"reach.v.04"},{"index":3,"sense_key":"reach%2:42:00::","synset_id":"reach.v.06"},{"index":4,"sense_key":"reach%2:33:00::","synset_id":"strive.v.02"},{"index":5,"sense_key":"reach%2:40:00::","synset_id":"pass.v.05"},{"index":6,"sense_key":"reach%2:38:00::","synset_id":"reach.v.02"},{"index":7,"sense_key":"reach%2:38:06::","synset_id":"reach.v.07"},{"index":8,"sense_key":"reach%2:41:00::","synset_id":"achieve.v.01"},{"index":9,"sense_key":"reach%2:35:00::","synset_id":"reach.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reach%2:38:01::","call_ids":["semeval2015.d000.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reach%2:38:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":189,"reasoning_output_tokens":173},"cost":{"currency":"USD","total_usd":0.008935,"input_uncached_usd":0.003265,"input_cached_usd":0.0,"output_usd":0.0056700000000000006,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.694839917006902}
{"item_id":"semeval2015.d000.s001.t010","gold_sense_keys":["recommendation%1:10:00::"],"candidates":[{"index":1,"sense_key":"recommendation%1:10:00::","synset_id":"recommendation.n.01"},{"index":2,"sense_key":"recommendation%1:07:00::","synset_id":"recommendation.n.03"},{"index":3,"sense_key":"recommendation%1:10:02::","synset_id":"recommendation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recommendation%1:10:00::","call_ids":["semeval2015.d000.s001.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recommendation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":363,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.0036750000000000003,"input_uncached_usd":0.0018150000000000002,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.587580666062422}
{"item_id":"semeval2015.d000.s001.t011","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":2,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":3,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":4,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":5,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:02::","call_ids":["semeval2015.d000.s001.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01839,"input_uncached_usd":0.00255,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.893473832984455}
{"item_id":"semeval2015.d000.s001.t012","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s001.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.00398,"input_uncached_usd":0.002,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.658089582924731}
{"item_id":"semeval2015.d000.s002.t000","gold_sense_keys":["need%2:34:00::","need%2:34:01::"],"candidates":[{"index":1,"sense_key":"need%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"need%2:34:01::","synset_id":"need.v.03"},{"index":3,"sense_key":"need%2:34:00::","synset_id":"want.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"need%2:34:01::","call_ids":["semeval2015.d000.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"need%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":380,"cached_input_tokens":0,"output_tokens":769,"reasoning_output_tokens":747},"cost":{"currency":"USD","total_usd":0.02497,"input_uncached_usd":0.0019000000000000002,"input_cached_usd":0.0,"output_usd":0.02307,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.826202750089578}
{"item_id":"semeval2015.d000.s002.t001","gold_sense_keys":["more%3:00:01::"],"candidates":[{"index":1,"sense_key":"more%3:00:02::","synset_id":"more.a.02"},{"index":2,"sense_key":"more%3:00:01::","synset_id":"more.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"more%3:00:01::","call_ids":["semeval2015.d000.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"more%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":355,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.003515,"input_uncached_usd":0.001775,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9714205419877544}
{"item_id":"semeval2015.d000.s002.t002","gold_sense_keys":["information%1:09:00::"],"candidates":[{"index":1,"sense_key":"information%1:10:02::","synset_id":"information.n.03"},{"index":2,"sense_key":"information%1:10:00::","synset_id":"information.n.01"},{"index":3,"sense_key":"information%1:09:00::","synset_id":"information.n.02"},{"index":4,"sense_key":"information%1:14:00::","synset_id":"data.n.01"},{"index":5,"sense_key":"information%1:07:00::","synset_id":"information.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"information%1:10:00::","call_ids":["semeval2015.d000.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"information%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.033424999999999996,"input_uncached_usd":0.0020450000000000004,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":17.621373833040707}
{"item_id":"semeval2015.d000.s002.t003","gold_sense_keys":["medical%3:00:00::"],"candidates":[{"index":1,"sense_key":"medical%3:01:02::","synset_id":"aesculapian.a.01"},{"index":2,"sense_key":"medical%3:00:00::","synset_id":"medical.a.02"},{"index":3,"sense_key":"medical%3:01:00::","synset_id":"medical.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medical%3:00:00::","call_ids":["semeval2015.d000.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medical%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":369,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017685,"input_uncached_usd":0.001845,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.261022208025679}
{"item_id":"semeval2015.d000.s002.t004","gold_sense_keys":["condition%1:26:05::"],"candidates":[{"index":1,"sense_key":"condition%1:26:05::","synset_id":"condition.n.06"},{"index":2,"sense_key":"condition%1:10:01::","synset_id":"condition.n.02"},{"index":3,"sense_key":"condition%1:26:01::","synset_id":"condition.n.03"},{"index":4,"sense_key":"condition%1:10:02::","synset_id":"condition.n.07"},{"index":5,"sense_key":"condition%1:26:00::","synset_id":"condition.n.01"},{"index":6,"sense_key":"condition%1:09:01::","synset_id":"condition.n.08"},{"index":7,"sense_key":"condition%1:26:02::","synset_id":"condition.n.05"},{"index":8,"sense_key":"condition%1:09:00::","synset_id":"circumstance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"condition%1:26:05::","call_ids":["semeval2015.d000.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"condition%1:26:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.0043300000000000005,"input_uncached_usd":0.00301,"input_cached_usd":0.0,"output_usd":0.00132,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4419284579344094}
{"item_id":"semeval2015.d000.s002.t005","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"},{"index":3,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":4,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d000.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":430,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.0041600000000000005,"input_uncached_usd":0.00215,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5879922080785036}
{"item_id":"semeval2015.d000.s002.t006","gold_sense_keys":["read%2:31:00::"],"candidates":[{"index":1,"sense_key":"read%2:31:06::","synset_id":"read.v.04"},{"index":2,"sense_key":"read%2:31:04::","synset_id":"understand.v.03"},{"index":3,"sense_key":"read%2:31:09::","synset_id":"take.v.06"},{"index":4,"sense_key":"read%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"read%2:31:00::","synset_id":"read.v.01"},{"index":6,"sense_key":"read%2:31:01::","synset_id":"read.v.03"},{"index":7,"sense_key":"read%2:31:05::","synset_id":"learn.v.04"},{"index":8,"sense_key":"read%2:31:02::","synset_id":"read.v.05"},{"index":9,"sense_key":"read%2:32:02::","synset_id":"read.v.08"},{"index":10,"sense_key":"read%2:31:08::","synset_id":"read.v.10"},{"index":11,"sense_key":"read%2:36:05::","synset_id":"read.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"read%2:31:00::","call_ids":["semeval2015.d000.s002.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"read%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":734,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.0064600000000000005,"input_uncached_usd":0.00367,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.911505208001472}
{"item_id":"semeval2015.d000.s002.t008","gold_sense_keys":["leaflet%1:10:00::"],"candidates":[{"index":1,"sense_key":"leaflet%1:10:00::","synset_id":"booklet.n.01"},{"index":2,"sense_key":"leaflet%1:08:00::","synset_id":"cusp.n.02"},{"index":3,"sense_key":"leaflet%1:20:00::","synset_id":"leaflet.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"leaflet%1:10:00::","call_ids":["semeval2015.d000.s002.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"leaflet%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":344,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.0031000000000000003,"input_uncached_usd":0.0017200000000000002,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3843751250533387}
{"item_id":"semeval2015.d000.s002.t010","gold_sense_keys":["part%1:09:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":2,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":3,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":4,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":5,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":6,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":7,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":8,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":9,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":10,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":11,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"part%1:24:00::","call_ids":["semeval2015.d000.s002.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"part%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":847,"cached_input_tokens":0,"output_tokens":1564,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.051155000000000006,"input_uncached_usd":0.004235,"input_cached_usd":0.0,"output_usd":0.04692,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":23.251713625038974}
{"item_id":"semeval2015.d000.s002.t011","gold_sense_keys":["contact%2:32:00::"],"candidates":[{"index":1,"sense_key":"contact%2:32:00::","synset_id":"reach.v.04"},{"index":2,"sense_key":"contact%2:35:03::","synset_id":"touch.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contact%2:32:00::","call_ids":["semeval2015.d000.s002.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contact%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":353,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.003265,"input_uncached_usd":0.001765,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4779523329343647}
{"item_id":"semeval2015.d000.s002.t012","gold_sense_keys":["doctor%1:18:00::"],"candidates":[{"index":1,"sense_key":"doctor%1:18:00::","synset_id":"doctor.n.01"},{"index":2,"sense_key":"doctor%1:04:00::","synset_id":"doctor.n.03"},{"index":3,"sense_key":"doctor%1:18:02::","synset_id":"doctor_of_the_church.n.01"},{"index":4,"sense_key":"doctor%1:18:01::","synset_id":"doctor.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"doctor%1:18:00::","call_ids":["semeval2015.d000.s002.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"doctor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":33,"reasoning_output_tokens":18},"cost":{"currency":"USD","total_usd":0.0033150000000000002,"input_uncached_usd":0.0023250000000000002,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2427621249807999}
{"item_id":"semeval2015.d000.s003.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"want%2:37:00::","call_ids":["semeval2015.d000.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.003875,"input_uncached_usd":0.002285,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.196954083046876}
{"item_id":"semeval2015.d000.s003.t001","gold_sense_keys":["more%3:00:01::"],"candidates":[{"index":1,"sense_key":"more%3:00:01::","synset_id":"more.a.01"},{"index":2,"sense_key":"more%3:00:02::","synset_id":"more.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%3:00:01::","call_ids":["semeval2015.d000.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":361,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":90},"cost":{"currency":"USD","total_usd":0.004955,"input_uncached_usd":0.0018050000000000002,"input_cached_usd":0.0,"output_usd":0.00315,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.536964208004065}
{"item_id":"semeval2015.d000.s003.t002","gold_sense_keys":["information%1:09:00::","information%1:14:00::"],"candidates":[{"index":1,"sense_key":"information%1:10:02::","synset_id":"information.n.03"},{"index":2,"sense_key":"information%1:07:00::","synset_id":"information.n.05"},{"index":3,"sense_key":"information%1:09:00::","synset_id":"information.n.02"},{"index":4,"sense_key":"information%1:10:00::","synset_id":"information.n.01"},{"index":5,"sense_key":"information%1:14:00::","synset_id":"data.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"information%1:09:00::","call_ids":["semeval2015.d000.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"information%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017915,"input_uncached_usd":0.002075,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.375965417013504}
{"item_id":"semeval2015.d000.s003.t003","gold_sense_keys":["basis%1:09:00::"],"candidates":[{"index":1,"sense_key":"basis%1:24:01::","synset_id":"basis.n.03"},{"index":2,"sense_key":"basis%1:24:00::","synset_id":"footing.n.02"},{"index":3,"sense_key":"basis%1:09:00::","synset_id":"basis.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"basis%1:09:00::","call_ids":["semeval2015.d000.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"basis%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":499,"reasoning_output_tokens":483},"cost":{"currency":"USD","total_usd":0.016985,"input_uncached_usd":0.0020150000000000003,"input_cached_usd":0.0,"output_usd":0.01497,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.836382917012088}
{"item_id":"semeval2015.d000.s003.t004","gold_sense_keys":["recommendation%1:10:00::"],"candidates":[{"index":1,"sense_key":"recommendation%1:10:00::","synset_id":"recommendation.n.01"},{"index":2,"sense_key":"recommendation%1:07:00::","synset_id":"recommendation.n.03"},{"index":3,"sense_key":"recommendation%1:10:02::","synset_id":"recommendation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recommendation%1:10:00::","call_ids":["semeval2015.d000.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recommendation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":394,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.00431,"input_uncached_usd":0.00197,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9024272080278024}
{"item_id":"semeval2015.d000.s003.t005","gold_sense_keys":["read%2:31:00::"],"candidates":[{"index":1,"sense_key":"read%2:32:02::","synset_id":"read.v.08"},{"index":2,"sense_key":"read%2:31:05::","synset_id":"learn.v.04"},{"index":3,"sense_key":"read%2:31:08::","synset_id":"read.v.10"},{"index":4,"sense_key":"read%2:31:02::","synset_id":"read.v.05"},{"index":5,"sense_key":"read%2:36:05::","synset_id":"read.v.09"},{"index":6,"sense_key":"read%2:31:09::","synset_id":"take.v.06"},{"index":7,"sense_key":"read%2:31:06::","synset_id":"read.v.04"},{"index":8,"sense_key":"read%2:31:04::","synset_id":"understand.v.03"},{"index":9,"sense_key":"read%2:31:01::","synset_id":"read.v.03"},{"index":10,"sense_key":"read%2:31:00::","synset_id":"read.v.01"},{"index":11,"sense_key":"read%2:42:00::","synset_id":"read.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"read%2:31:00::","call_ids":["semeval2015.d000.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"read%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":740,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.00628,"input_uncached_usd":0.0037,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9954362919088453}
{"item_id":"semeval2015.d000.s003.t007","gold_sense_keys":["discussion%1:10:02::"],"candidates":[{"index":1,"sense_key":"discussion%1:10:00::","synset_id":"discussion.n.02"},{"index":2,"sense_key":"discussion%1:10:02::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"discussion%1:10:02::","call_ids":["semeval2015.d000.s003.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"discussion%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":367,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.004745,"input_uncached_usd":0.001835,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.878367832978256}
{"item_id":"semeval2015.d000.s003.t009","gold_sense_keys":["part%1:09:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":2,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":3,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":4,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":5,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":6,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":7,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":8,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":9,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"},{"index":10,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":11,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":12,"sense_key":"part%1:09:01::","synset_id":"character.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"part%1:24:00::","call_ids":["semeval2015.d000.s003.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"part%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.035644999999999996,"input_uncached_usd":0.0042650000000000006,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":16.405474082916044}
{"item_id":"semeval2015.d000.s005.t000","gold_sense_keys":["powder%1:06:01::","powder%1:27:00::"],"candidates":[{"index":1,"sense_key":"powder%1:27:00::","synset_id":"powder.n.01"},{"index":2,"sense_key":"powder%1:27:01::","synset_id":"gunpowder.n.01"},{"index":3,"sense_key":"powder%1:06:01::","synset_id":"powder.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"powder%1:06:01::","call_ids":["semeval2015.d000.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"powder%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":119},"cost":{"currency":"USD","total_usd":0.006185000000000001,"input_uncached_usd":0.002135,"input_cached_usd":0.0,"output_usd":0.00405,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.549502332927659}
{"item_id":"semeval2015.d000.s005.t002","gold_sense_keys":["solution%1:27:00::"],"candidates":[{"index":1,"sense_key":"solution%1:04:00::","synset_id":"solution.n.05"},{"index":2,"sense_key":"solution%1:14:00::","synset_id":"solution.n.04"},{"index":3,"sense_key":"solution%1:27:00::","synset_id":"solution.n.01"},{"index":4,"sense_key":"solution%1:09:00::","synset_id":"solution.n.03"},{"index":5,"sense_key":"solution%1:10:00::","synset_id":"solution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"solution%1:27:00::","call_ids":["semeval2015.d000.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"solution%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.00423,"input_uncached_usd":0.00249,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.814440582995303}
{"item_id":"semeval2015.d000.s005.t003","gold_sense_keys":["infusion%1:04:00::"],"candidates":[{"index":1,"sense_key":"infusion%1:27:00::","synset_id":"infusion.n.01"},{"index":2,"sense_key":"infusion%1:04:01::","synset_id":"infusion.n.04"},{"index":3,"sense_key":"infusion%1:04:00::","synset_id":"infusion.n.03"},{"index":4,"sense_key":"infusion%1:22:00::","synset_id":"infusion.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"infusion%1:04:00::","call_ids":["semeval2015.d000.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"infusion%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.00363,"input_uncached_usd":0.00231,"input_cached_usd":0.0,"output_usd":0.00132,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.314301333972253}
{"item_id":"semeval2015.d000.s005.t004","gold_sense_keys":["drip%2:35:10::"],"candidates":[{"index":1,"sense_key":"drip%2:35:10::","synset_id":"dribble.v.02"},{"index":2,"sense_key":"drip%2:38:00::","synset_id":"drip.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drip%2:35:10::","call_ids":["semeval2015.d000.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drip%2:35:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":361,"cached_input_tokens":0,"output_tokens":2548,"reasoning_output_tokens":2506},"cost":{"currency":"USD","total_usd":0.078245,"input_uncached_usd":0.0018050000000000002,"input_cached_usd":0.0,"output_usd":0.07644000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":41.290206416044384}
{"item_id":"semeval2015.d000.s005.t005","gold_sense_keys":["vein%1:08:00::"],"candidates":[{"index":1,"sense_key":"vein%1:10:00::","synset_id":"vein.n.02"},{"index":2,"sense_key":"vein%1:17:00::","synset_id":"vein.n.04"},{"index":3,"sense_key":"vein%1:20:00::","synset_id":"vein.n.03"},{"index":4,"sense_key":"vein%1:08:00::","synset_id":"vein.n.01"},{"index":5,"sense_key":"vein%1:05:00::","synset_id":"vein.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"vein%1:08:00::","call_ids":["semeval2015.d000.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"vein%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.004055,"input_uncached_usd":0.002465,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3950228339526802}
{"item_id":"semeval2015.d000.s006.t000","gold_sense_keys":["contain%2:42:00::"],"candidates":[{"index":1,"sense_key":"contain%2:42:06::","synset_id":"contain.v.04"},{"index":2,"sense_key":"contain%2:42:14::","synset_id":"contain.v.05"},{"index":3,"sense_key":"contain%2:42:13::","synset_id":"hold.v.11"},{"index":4,"sense_key":"contain%2:33:00::","synset_id":"check.v.18"},{"index":5,"sense_key":"contain%2:41:00::","synset_id":"control.v.02"},{"index":6,"sense_key":"contain%2:42:00::","synset_id":"incorporate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"contain%2:42:00::","call_ids":["semeval2015.d000.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"contain%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.00542,"input_uncached_usd":0.0029300000000000003,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.938823042088188}
{"item_id":"semeval2015.d000.s006.t001","gold_sense_keys":["active%3:00:07::"],"candidates":[{"index":1,"sense_key":"active%5:00:00:existent:00","synset_id":"active.s.03"},{"index":2,"sense_key":"active%3:00:04::","synset_id":"active.a.13"},{"index":3,"sense_key":"active%3:00:08::","synset_id":"active.a.14"},{"index":4,"sense_key":"active%3:00:02::","synset_id":"active.a.01"},{"index":5,"sense_key":"active%3:00:09::","synset_id":"active.a.10"},{"index":6,"sense_key":"active%3:00:07::","synset_id":"active.a.06"},{"index":7,"sense_key":"active%3:00:05::","synset_id":"active.a.12"},{"index":8,"sense_key":"active%5:00:00:involved:00","synset_id":"active.s.02"},{"index":9,"sense_key":"active%3:00:03::","synset_id":"active.a.03"},{"index":10,"sense_key":"active%3:00:06::","synset_id":"active.a.07"},{"index":11,"sense_key":"active%5:00:00:operational:00","synset_id":"active.s.01"},{"index":12,"sense_key":"active%3:00:00::","synset_id":"active.a.11"},{"index":13,"sense_key":"active%3:00:01::","synset_id":"active.a.05"},{"index":14,"sense_key":"active%3:00:10::","synset_id":"active.a.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"active%3:00:07::","call_ids":["semeval2015.d000.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"active%3:00:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":830,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00607,"input_uncached_usd":0.00415,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9850965839577839}
{"item_id":"semeval2015.d000.s006.t002","gold_sense_keys":["substance%1:03:01::"],"candidates":[{"index":1,"sense_key":"substance%1:09:00::","synset_id":"meaning.n.02"},{"index":2,"sense_key":"substance%1:03:01::","synset_id":"substance.n.07"},{"index":3,"sense_key":"substance%1:09:01::","synset_id":"kernel.n.03"},{"index":4,"sense_key":"substance%1:03:00::","synset_id":"substance.n.01"},{"index":5,"sense_key":"substance%1:10:00::","synset_id":"message.n.02"},{"index":6,"sense_key":"substance%1:27:00::","synset_id":"substance.n.04"},{"index":7,"sense_key":"substance%1:21:00::","synset_id":"means.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"substance%1:03:01::","call_ids":["semeval2015.d000.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"substance%1:03:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":6874,"reasoning_output_tokens":6782},"cost":{"currency":"USD","total_usd":0.20908500000000002,"input_uncached_usd":0.0028650000000000004,"input_cached_usd":0.0,"output_usd":0.20622000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":135.25935266597662}
{"item_id":"semeval2015.d000.s007.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":2,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":3,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":4,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":5,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d000.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0047350000000000005,"input_uncached_usd":0.0028450000000000003,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.571021959069185}
{"item_id":"semeval2015.d000.s008.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":2,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":3,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":4,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:34:01::","synset_id":"use.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d000.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.005045,"input_uncached_usd":0.003095,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3842345000011846}
{"item_id":"semeval2015.d000.s008.t001","gold_sense_keys":["treat%2:29:00::"],"candidates":[{"index":1,"sense_key":"treat%2:32:03::","synset_id":"treat.v.07"},{"index":2,"sense_key":"treat%2:34:00::","synset_id":"regale.v.01"},{"index":3,"sense_key":"treat%2:31:00::","synset_id":"treat.v.08"},{"index":4,"sense_key":"treat%2:41:00::","synset_id":"treat.v.01"},{"index":5,"sense_key":"treat%2:40:00::","synset_id":"treat.v.05"},{"index":6,"sense_key":"treat%2:29:00::","synset_id":"treat.v.03"},{"index":7,"sense_key":"treat%2:30:01::","synset_id":"process.v.01"},{"index":8,"sense_key":"treat%2:32:00::","synset_id":"cover.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"treat%2:29:00::","call_ids":["semeval2015.d000.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"treat%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":709,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.005345000000000001,"input_uncached_usd":0.0035450000000000004,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.634255083044991}
{"item_id":"semeval2015.d000.s008.t002","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":2,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":3,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":4,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":5,"sense_key":"type%1:06:00::","synset_id":"type.n.06"},{"index":6,"sense_key":"type%1:10:00::","synset_id":"type.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2015.d000.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00476,"input_uncached_usd":0.00287,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7815889999037609}
{"item_id":"semeval2015.d000.s008.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":4,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":5,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d000.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.004425,"input_uncached_usd":0.0026850000000000003,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0699023329652846}
{"item_id":"semeval2015.d000.s008.t006","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":4,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d000.s008.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.00412,"input_uncached_usd":0.00268,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.567086874973029}
{"item_id":"semeval2015.d000.s008.t007","gold_sense_keys":["lining%1:06:00::"],"candidates":[{"index":1,"sense_key":"lining%1:04:00::","synset_id":"lining.n.04"},{"index":2,"sense_key":"lining%1:06:00::","synset_id":"lining.n.01"},{"index":3,"sense_key":"lining%1:06:01::","synset_id":"liner.n.03"},{"index":4,"sense_key":"lining%1:04:01::","synset_id":"lining.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"lining%1:06:00::","call_ids":["semeval2015.d000.s008.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"lining%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":96},"cost":{"currency":"USD","total_usd":0.005665,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.00336,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.560181458014995}
{"item_id":"semeval2015.d000.s008.t010","gold_sense_keys":["cause%2:36:00::"],"candidates":[{"index":1,"sense_key":"cause%2:32:00::","synset_id":"induce.v.02"},{"index":2,"sense_key":"cause%2:36:00::","synset_id":"cause.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cause%2:36:00::","call_ids":["semeval2015.d000.s008.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cause%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.00449,"input_uncached_usd":0.0022400000000000002,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.174675334012136}
{"item_id":"semeval2015.d000.s009.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":2,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":5,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":6,"sense_key":"use%2:41:14::","synset_id":"use.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d000.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.0054,"input_uncached_usd":0.0031500000000000005,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2690541250631213}
{"item_id":"semeval2015.d000.s009.t003","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":2,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":3,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":33,"reasoning_output_tokens":18},"cost":{"currency":"USD","total_usd":0.0035900000000000003,"input_uncached_usd":0.0026000000000000003,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4312872498994693}
{"item_id":"semeval2015.d000.s009.t004","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":4,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":5,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d000.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.004185,"input_uncached_usd":0.002745,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2445325839798898}
{"item_id":"semeval2015.d000.s009.t005","gold_sense_keys":["remove%2:30:00::"],"candidates":[{"index":1,"sense_key":"remove%2:30:00::","synset_id":"remove.v.01"},{"index":2,"sense_key":"remove%2:38:00::","synset_id":"remove.v.05"},{"index":3,"sense_key":"remove%2:41:01::","synset_id":"murder.v.01"},{"index":4,"sense_key":"remove%2:30:01::","synset_id":"absent.v.01"},{"index":5,"sense_key":"remove%2:41:00::","synset_id":"remove.v.02"},{"index":6,"sense_key":"remove%2:41:02::","synset_id":"take_out.v.01"},{"index":7,"sense_key":"remove%2:40:00::","synset_id":"get_rid_of.v.01"},{"index":8,"sense_key":"remove%2:30:02::","synset_id":"remove.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"remove%2:30:00::","call_ids":["semeval2015.d000.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"remove%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.006585000000000001,"input_uncached_usd":0.0034950000000000003,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6212206250056624}
{"item_id":"semeval2015.d000.s009.t006","gold_sense_keys":["surgery%1:04:01::"],"candidates":[{"index":1,"sense_key":"surgery%1:09:00::","synset_id":"surgery.n.01"},{"index":2,"sense_key":"surgery%1:06:01::","synset_id":"surgery.n.02"},{"index":3,"sense_key":"surgery%1:04:01::","synset_id":"operation.n.06"},{"index":4,"sense_key":"surgery%1:06:00::","synset_id":"operating_room.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"surgery%1:04:01::","call_ids":["semeval2015.d000.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"surgery%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.004305,"input_uncached_usd":0.002775,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7147971669910476}
{"item_id":"semeval2015.d000.s009.t007","gold_sense_keys":["alone%5:00:00:exclusive:00"],"candidates":[{"index":1,"sense_key":"alone%5:00:00:incomparable:00","synset_id":"alone.s.04"},{"index":2,"sense_key":"alone%5:00:00:unsocial:00","synset_id":"alone.s.01"},{"index":3,"sense_key":"alone%5:00:00:exclusive:00","synset_id":"alone.s.03"},{"index":4,"sense_key":"alone%5:00:00:unaccompanied:00","synset_id":"alone.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"alone%5:00:00:exclusive:00","call_ids":["semeval2015.d000.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"alone%5:00:00:exclusive:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.0046,"input_uncached_usd":0.0027400000000000002,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5902891249861568}
{"item_id":"semeval2015.d000.s009.t009","gold_sense_keys":["spread%2:35:07::"],"candidates":[{"index":1,"sense_key":"spread%2:35:00::","synset_id":"spread.v.01"},{"index":2,"sense_key":"spread%2:35:12::","synset_id":"spread.v.09"},{"index":3,"sense_key":"spread%2:32:02::","synset_id":"go_around.v.02"},{"index":4,"sense_key":"spread%2:38:02::","synset_id":"diffuse.v.01"},{"index":5,"sense_key":"spread%2:32:00::","synset_id":"circulate.v.02"},{"index":6,"sense_key":"spread%2:35:13::","synset_id":"spread.v.10"},{"index":7,"sense_key":"spread%2:38:00::","synset_id":"spread.v.07"},{"index":8,"sense_key":"spread%2:42:00::","synset_id":"spread.v.03"},{"index":9,"sense_key":"spread%2:35:03::","synset_id":"unfold.v.04"},{"index":10,"sense_key":"spread%2:35:07::","synset_id":"spread.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"spread%2:35:07::","call_ids":["semeval2015.d000.s009.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"spread%2:35:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":95},"cost":{"currency":"USD","total_usd":0.007095000000000001,"input_uncached_usd":0.003765,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.950527291977778}
{"item_id":"semeval2015.d000.s009.t010","gold_sense_keys":["likely%3:00:00::"],"candidates":[{"index":1,"sense_key":"likely%5:00:00:credible:00","synset_id":"likely.s.02"},{"index":2,"sense_key":"likely%3:00:00::","synset_id":"likely.a.01"},{"index":3,"sense_key":"likely%3:00:04::","synset_id":"probable.a.01"},{"index":4,"sense_key":"likely%5:00:00:prospective:00","synset_id":"likely.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"likely%3:00:00::","call_ids":["semeval2015.d000.s009.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"likely%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":88},"cost":{"currency":"USD","total_usd":0.005690000000000001,"input_uncached_usd":0.0025700000000000002,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6167124999919906}
{"item_id":"semeval2015.d000.s009.t011","gold_sense_keys":["spread%2:35:07::"],"candidates":[{"index":1,"sense_key":"spread%2:35:07::","synset_id":"spread.v.02"},{"index":2,"sense_key":"spread%2:32:00::","synset_id":"circulate.v.02"},{"index":3,"sense_key":"spread%2:35:03::","synset_id":"unfold.v.04"},{"index":4,"sense_key":"spread%2:35:12::","synset_id":"spread.v.09"},{"index":5,"sense_key":"spread%2:38:00::","synset_id":"spread.v.07"},{"index":6,"sense_key":"spread%2:32:02::","synset_id":"go_around.v.02"},{"index":7,"sense_key":"spread%2:42:00::","synset_id":"spread.v.03"},{"index":8,"sense_key":"spread%2:38:02::","synset_id":"diffuse.v.01"},{"index":9,"sense_key":"spread%2:35:13::","synset_id":"spread.v.10"},{"index":10,"sense_key":"spread%2:35:00::","synset_id":"spread.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"spread%2:35:07::","call_ids":["semeval2015.d000.s009.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"spread%2:35:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":120},"cost":{"currency":"USD","total_usd":0.007850000000000001,"input_uncached_usd":0.0037700000000000003,"input_cached_usd":0.0,"output_usd":0.00408,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6046887919073924}
{"item_id":"semeval2015.d000.s009.t012","gold_sense_keys":["easily%4:02:01::"],"candidates":[{"index":1,"sense_key":"easily%4:02:05::","synset_id":"well.r.03"},{"index":2,"sense_key":"easily%4:02:01::","synset_id":"easily.r.01"},{"index":3,"sense_key":"easily%4:02:06::","synset_id":"easily.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"easily%4:02:01::","call_ids":["semeval2015.d000.s009.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"easily%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":113},"cost":{"currency":"USD","total_usd":0.00622,"input_uncached_usd":0.00235,"input_cached_usd":0.0,"output_usd":0.00387,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.56539987504948}
{"item_id":"semeval2015.d000.s009.t013","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":2,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":3,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":4,"sense_key":"other%3:00:00::","synset_id":"other.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"other%3:00:00::","call_ids":["semeval2015.d000.s009.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00432,"input_uncached_usd":0.0026100000000000003,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7028095839777961}
{"item_id":"semeval2015.d000.s009.t014","gold_sense_keys":["part%1:15:00::","part%1:17:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"},{"index":2,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":3,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":4,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":5,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":6,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":7,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":8,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":9,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":12,"sense_key":"part%1:09:02::","synset_id":"part.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"part%1:15:00::","call_ids":["semeval2015.d000.s009.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"part%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":941,"cached_input_tokens":0,"output_tokens":2579,"reasoning_output_tokens":2538},"cost":{"currency":"USD","total_usd":0.082075,"input_uncached_usd":0.004705,"input_cached_usd":0.0,"output_usd":0.07737000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":48.09464595897589}
{"item_id":"semeval2015.d000.s009.t015","gold_sense_keys":["body%1:08:00::"],"candidates":[{"index":1,"sense_key":"body%1:08:02::","synset_id":"body.n.03"},{"index":2,"sense_key":"body%1:06:01::","synset_id":"soundbox.n.01"},{"index":3,"sense_key":"body%1:10:00::","synset_id":"body.n.08"},{"index":4,"sense_key":"body%1:07:00::","synset_id":"consistency.n.01"},{"index":5,"sense_key":"body%1:07:01::","synset_id":"body.n.09"},{"index":6,"sense_key":"body%1:14:02::","synset_id":"body.n.06"},{"index":7,"sense_key":"body%1:14:00::","synset_id":"body.n.02"},{"index":8,"sense_key":"body%1:17:00::","synset_id":"body.n.04"},{"index":9,"sense_key":"body%1:06:00::","synset_id":"body.n.11"},{"index":10,"sense_key":"body%1:08:01::","synset_id":"torso.n.01"},{"index":11,"sense_key":"body%1:08:00::","synset_id":"body.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"body%1:08:00::","call_ids":["semeval2015.d000.s009.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"body%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":797,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.005815000000000001,"input_uncached_usd":0.003985000000000001,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4171837499598041}
{"item_id":"semeval2015.d000.s009.t016","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"},{"index":2,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s009.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":33,"reasoning_output_tokens":17},"cost":{"currency":"USD","total_usd":0.0031650000000000003,"input_uncached_usd":0.0021750000000000003,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.087277084006928}
{"item_id":"semeval2015.d000.s009.t020","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s009.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.0043100000000000005,"input_uncached_usd":0.0026000000000000003,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.545001499936916}
{"item_id":"semeval2015.d000.s009.t021","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":2,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":5,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d000.s009.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.004450000000000001,"input_uncached_usd":0.0027400000000000002,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.062543833977543}
{"item_id":"semeval2015.d000.s009.t022","gold_sense_keys":["advanced%5:00:00:late:02"],"candidates":[{"index":1,"sense_key":"advanced%5:00:00:late:00","synset_id":"advanced.s.06"},{"index":2,"sense_key":"advanced%5:00:00:progressive:01","synset_id":"advanced.s.03"},{"index":3,"sense_key":"advanced%5:00:02:high:02","synset_id":"advanced.s.04"},{"index":4,"sense_key":"advanced%5:00:00:front:00","synset_id":"advance.s.02"},{"index":5,"sense_key":"advanced%5:00:00:civilized:00","synset_id":"advanced.s.07"},{"index":6,"sense_key":"advanced%5:00:00:high-tech:00","synset_id":"advanced.s.05"},{"index":7,"sense_key":"advanced%5:00:00:late:02","synset_id":"advanced.s.02"},{"index":8,"sense_key":"advanced%5:00:00:precocious:00","synset_id":"advanced.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"advanced%5:00:00:late:02","call_ids":["semeval2015.d000.s009.t022__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"advanced%5:00:00:late:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":716,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.00613,"input_uncached_usd":0.0035800000000000003,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.438106249901466}
{"item_id":"semeval2015.d000.s009.t024","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":5,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"cell%1:03:00::","call_ids":["semeval2015.d000.s009.t024__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.005655,"input_uncached_usd":0.003105,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.305367457913235}
{"item_id":"semeval2015.d000.s009.t027","gold_sense_keys":["affect%2:29:00::"],"candidates":[{"index":1,"sense_key":"affect%2:42:00::","synset_id":"involve.v.01"},{"index":2,"sense_key":"affect%2:30:00::","synset_id":"affect.v.01"},{"index":3,"sense_key":"affect%2:37:00::","synset_id":"affect.v.05"},{"index":4,"sense_key":"affect%2:32:00::","synset_id":"feign.v.01"},{"index":5,"sense_key":"affect%2:29:00::","synset_id":"affect.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"affect%2:29:00::","call_ids":["semeval2015.d000.s009.t027__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"affect%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":576,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01872,"input_uncached_usd":0.00288,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.207885916926898}
{"item_id":"semeval2015.d000.s010.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":2,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":5,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":6,"sense_key":"use%2:41:14::","synset_id":"use.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d000.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.00584,"input_uncached_usd":0.00317,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.103699458995834}
{"item_id":"semeval2015.d000.s010.t002","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.0036950000000000004,"input_uncached_usd":0.0021950000000000003,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3182206249330193}
{"item_id":"semeval2015.d000.s010.t004","gold_sense_keys":["treat%2:29:00::"],"candidates":[{"index":1,"sense_key":"treat%2:31:00::","synset_id":"treat.v.08"},{"index":2,"sense_key":"treat%2:29:00::","synset_id":"treat.v.03"},{"index":3,"sense_key":"treat%2:32:00::","synset_id":"cover.v.05"},{"index":4,"sense_key":"treat%2:40:00::","synset_id":"treat.v.05"},{"index":5,"sense_key":"treat%2:41:00::","synset_id":"treat.v.01"},{"index":6,"sense_key":"treat%2:34:00::","synset_id":"regale.v.01"},{"index":7,"sense_key":"treat%2:32:03::","synset_id":"treat.v.07"},{"index":8,"sense_key":"treat%2:30:01::","synset_id":"process.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"treat%2:29:00::","call_ids":["semeval2015.d000.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"treat%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.006345,"input_uncached_usd":0.003615,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.489967082976364}
{"item_id":"semeval2015.d000.s010.t005","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s010.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.0038150000000000002,"input_uncached_usd":0.0021950000000000003,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6718163749901578}
{"item_id":"semeval2015.d000.s010.t007","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":2,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":3,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":4,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":5,"sense_key":"type%1:06:00::","synset_id":"type.n.06"},{"index":6,"sense_key":"type%1:14:01::","synset_id":"type.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2015.d000.s010.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.0044150000000000005,"input_uncached_usd":0.002945,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.707465457962826}
{"item_id":"semeval2015.d000.s010.t009","gold_sense_keys":["past%1:28:00::"],"candidates":[{"index":1,"sense_key":"past%1:10:00::","synset_id":"past.n.03"},{"index":2,"sense_key":"past%1:28:00::","synset_id":"past.n.01"},{"index":3,"sense_key":"past%1:28:01::","synset_id":"past.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"past%1:28:00::","call_ids":["semeval2015.d000.s010.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"past%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.004695,"input_uncached_usd":0.0023250000000000002,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.255434292019345}
{"item_id":"semeval2015.d000.s011.t000","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":3,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"},{"index":4,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.00459,"input_uncached_usd":0.00255,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4980474170297384}
{"item_id":"semeval2015.d000.s011.t001","gold_sense_keys":["only%4:02:01::"],"candidates":[{"index":1,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":2,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":3,"sense_key":"only%4:02:05::","synset_id":"only.r.07"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":6,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":7,"sense_key":"only%4:02:00::","synset_id":"only.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"only%4:02:01::","call_ids":["semeval2015.d000.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"only%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":333,"reasoning_output_tokens":317},"cost":{"currency":"USD","total_usd":0.013075,"input_uncached_usd":0.003085,"input_cached_usd":0.0,"output_usd":0.00999,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.21857787494082}
{"item_id":"semeval2015.d000.s011.t002","gold_sense_keys":["obtain%2:40:00::"],"candidates":[{"index":1,"sense_key":"obtain%2:42:00::","synset_id":"prevail.v.02"},{"index":2,"sense_key":"obtain%2:40:00::","synset_id":"obtain.v.01"},{"index":3,"sense_key":"obtain%2:30:00::","synset_id":"receive.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"obtain%2:40:00::","call_ids":["semeval2015.d000.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"obtain%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004125,"input_uncached_usd":0.0023250000000000002,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.449757375055924}
{"item_id":"semeval2015.d000.s011.t003","gold_sense_keys":["prescription%1:10:01::"],"candidates":[{"index":1,"sense_key":"prescription%1:10:00::","synset_id":"prescription.n.01"},{"index":2,"sense_key":"prescription%1:10:02::","synset_id":"prescription.n.03"},{"index":3,"sense_key":"prescription%1:10:01::","synset_id":"prescription.n.04"},{"index":4,"sense_key":"prescription%1:06:00::","synset_id":"prescription_drug.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"prescription%1:10:01::","call_ids":["semeval2015.d000.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"prescription%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.00578,"input_uncached_usd":0.0025700000000000002,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6060542919440195}
{"item_id":"semeval2015.d000.s012.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":2,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":3,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":4,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":5,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":6,"sense_key":"use%2:34:00::","synset_id":"use.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d000.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":193,"reasoning_output_tokens":177},"cost":{"currency":"USD","total_usd":0.008955000000000001,"input_uncached_usd":0.0031650000000000003,"input_cached_usd":0.0,"output_usd":0.00579,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4470656669000164}
{"item_id":"semeval2015.d000.s013.t000","gold_sense_keys":["only%4:02:03::"],"candidates":[{"index":1,"sense_key":"only%4:02:05::","synset_id":"only.r.07"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":5,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"only%4:02:01::","call_ids":["semeval2015.d000.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"only%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019105,"input_uncached_usd":0.003265,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.347227542079054}
{"item_id":"semeval2015.d000.s013.t001","gold_sense_keys":["give%2:40:13::"],"candidates":[{"index":1,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":2,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":3,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":4,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":5,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":6,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":7,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":8,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":9,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":10,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":11,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":12,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":13,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":14,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":15,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":16,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":17,"sense_key":"give%2:29:10::","synset_id":"give.v.44"},{"index":18,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":19,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":20,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":21,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":22,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":23,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":24,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":25,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":26,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":27,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":28,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":29,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":30,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":31,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":32,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":33,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":34,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":35,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":36,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":37,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":38,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":39,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":40,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":41,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":42,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":43,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":44,"sense_key":"give%2:32:14::","synset_id":"give.v.38"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"give%2:40:13::","call_ids":["semeval2015.d000.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"give%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2122,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.01259,"input_uncached_usd":0.010610000000000001,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.007402708986774}
{"item_id":"semeval2015.d000.s013.t003","gold_sense_keys":["doctor%1:18:00::"],"candidates":[{"index":1,"sense_key":"doctor%1:04:00::","synset_id":"doctor.n.03"},{"index":2,"sense_key":"doctor%1:18:00::","synset_id":"doctor.n.01"},{"index":3,"sense_key":"doctor%1:18:02::","synset_id":"doctor_of_the_church.n.01"},{"index":4,"sense_key":"doctor%1:18:01::","synset_id":"doctor.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"doctor%1:18:00::","call_ids":["semeval2015.d000.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"doctor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.00436,"input_uncached_usd":0.0029200000000000003,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.51337408402469}
{"item_id":"semeval2015.d000.s013.t004","gold_sense_keys":["use%1:04:00::"],"candidates":[{"index":1,"sense_key":"use%1:04:01::","synset_id":"habit.n.02"},{"index":2,"sense_key":"use%1:22:00::","synset_id":"consumption.n.03"},{"index":3,"sense_key":"use%1:07:02::","synset_id":"use.n.03"},{"index":4,"sense_key":"use%1:07:03::","synset_id":"use.n.07"},{"index":5,"sense_key":"use%1:07:00::","synset_id":"function.n.02"},{"index":6,"sense_key":"use%1:04:00::","synset_id":"use.n.01"},{"index":7,"sense_key":"use%1:04:02::","synset_id":"manipulation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"use%1:04:00::","call_ids":["semeval2015.d000.s013.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"use%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.006620000000000001,"input_uncached_usd":0.0036500000000000005,"input_cached_usd":0.0,"output_usd":0.00297,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4778541669948027}
{"item_id":"semeval2015.d000.s014.t000","gold_sense_keys":["dose%1:06:00::"],"candidates":[{"index":1,"sense_key":"dose%1:26:00::","synset_id":"venereal_disease.n.01"},{"index":2,"sense_key":"dose%1:06:00::","synset_id":"dose.n.01"},{"index":3,"sense_key":"dose%1:06:01::","synset_id":"acid.n.02"},{"index":4,"sense_key":"dose%1:23:00::","synset_id":"dose.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dose%1:06:00::","call_ids":["semeval2015.d000.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dose%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":312,"reasoning_output_tokens":296},"cost":{"currency":"USD","total_usd":0.011985,"input_uncached_usd":0.002625,"input_cached_usd":0.0,"output_usd":0.00936,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.547339875018224}
{"item_id":"semeval2015.d000.s014.t001","gold_sense_keys":["mg%1:23:00::"],"candidates":[{"index":1,"sense_key":"mg%1:27:00::","synset_id":"magnesium.n.01"},{"index":2,"sense_key":"mg%1:23:00::","synset_id":"milligram.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mg%1:23:00::","call_ids":["semeval2015.d000.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mg%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0040100000000000005,"input_uncached_usd":0.0023000000000000004,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2139784580795094}
{"item_id":"semeval2015.d000.s014.t003","gold_sense_keys":["body%1:08:00::"],"candidates":[{"index":1,"sense_key":"body%1:06:01::","synset_id":"soundbox.n.01"},{"index":2,"sense_key":"body%1:08:01::","synset_id":"torso.n.01"},{"index":3,"sense_key":"body%1:08:00::","synset_id":"body.n.01"},{"index":4,"sense_key":"body%1:07:01::","synset_id":"body.n.09"},{"index":5,"sense_key":"body%1:14:02::","synset_id":"body.n.06"},{"index":6,"sense_key":"body%1:17:00::","synset_id":"body.n.04"},{"index":7,"sense_key":"body%1:14:00::","synset_id":"body.n.02"},{"index":8,"sense_key":"body%1:07:00::","synset_id":"consistency.n.01"},{"index":9,"sense_key":"body%1:06:00::","synset_id":"body.n.11"},{"index":10,"sense_key":"body%1:10:00::","synset_id":"body.n.08"},{"index":11,"sense_key":"body%1:08:02::","synset_id":"body.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"body%1:08:00::","call_ids":["semeval2015.d000.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"body%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.006565,"input_uncached_usd":0.004015,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.636827583075501}
{"item_id":"semeval2015.d000.s014.t005","gold_sense_keys":["calculate%2:31:00::"],"candidates":[{"index":1,"sense_key":"calculate%2:32:01::","synset_id":"forecast.v.01"},{"index":2,"sense_key":"calculate%2:31:05::","synset_id":"count.v.08"},{"index":3,"sense_key":"calculate%2:31:00::","synset_id":"calculate.v.01"},{"index":4,"sense_key":"calculate%2:31:03::","synset_id":"calculate.v.02"},{"index":5,"sense_key":"calculate%2:40:00::","synset_id":"account.v.02"},{"index":6,"sense_key":"calculate%2:31:02::","synset_id":"calculate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"calculate%2:31:00::","call_ids":["semeval2015.d000.s014.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"calculate%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.004505,"input_uncached_usd":0.0028250000000000003,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.680419374955818}
{"item_id":"semeval2015.d000.s014.t006","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":2,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":3,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":4,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":5,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d000.s014.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.00537,"input_uncached_usd":0.00318,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6195202079834417}
{"item_id":"semeval2015.d000.s014.t007","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"},{"index":2,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s014.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.004035,"input_uncached_usd":0.0022050000000000004,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8134144169744104}
{"item_id":"semeval2015.d000.s014.t008","gold_sense_keys":["height%1:07:01::"],"candidates":[{"index":1,"sense_key":"height%1:26:00::","synset_id":"acme.n.01"},{"index":2,"sense_key":"height%1:07:01::","synset_id":"stature.n.02"},{"index":3,"sense_key":"height%1:07:02::","synset_id":"altitude.n.01"},{"index":4,"sense_key":"height%1:07:00::","synset_id":"height.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"height%1:07:01::","call_ids":["semeval2015.d000.s014.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"height%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.00465,"input_uncached_usd":0.00267,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7737167079467326}
{"item_id":"semeval2015.d000.s014.t009","gold_sense_keys":["weight%1:07:00::"],"candidates":[{"index":1,"sense_key":"weight%1:07:00::","synset_id":"weight.n.01"},{"index":2,"sense_key":"weight%1:23:02::","synset_id":"weight.n.08"},{"index":3,"sense_key":"weight%1:23:01::","synset_id":"system_of_weights.n.01"},{"index":4,"sense_key":"weight%1:06:00::","synset_id":"weight.n.04"},{"index":5,"sense_key":"weight%1:07:01::","synset_id":"weight.n.03"},{"index":6,"sense_key":"weight%1:23:00::","synset_id":"weight_unit.n.02"},{"index":7,"sense_key":"weight%1:06:01::","synset_id":"weight.n.02"},{"index":8,"sense_key":"weight%1:12:00::","synset_id":"weight.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"weight%1:07:00::","call_ids":["semeval2015.d000.s014.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"weight%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":117},"cost":{"currency":"USD","total_usd":0.00723,"input_uncached_usd":0.0032700000000000003,"input_cached_usd":0.0,"output_usd":0.00396,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.863491249969229}
{"item_id":"semeval2015.d000.s015.t000","gold_sense_keys":["give%2:40:13::"],"candidates":[{"index":1,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":2,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":3,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":4,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":5,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":6,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":7,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":8,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":9,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":10,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":11,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":12,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":13,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":14,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":15,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":16,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":17,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":18,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":19,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":20,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":21,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":22,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":23,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":24,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":25,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":26,"sense_key":"give%2:29:10::","synset_id":"give.v.44"},{"index":27,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":28,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":29,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":32,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":33,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":34,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":35,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":36,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":37,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":38,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":39,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":40,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":41,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":42,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":43,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":44,"sense_key":"give%2:40:14::","synset_id":"give.v.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":31,"chosen_sense_key":"give%2:40:13::","call_ids":["semeval2015.d000.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":31,"predicted_sense_key":"give%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2055,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.013005000000000001,"input_uncached_usd":0.010275000000000001,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8930474579101428}
{"item_id":"semeval2015.d000.s015.t001","gold_sense_keys":["once%4:02:00::"],"candidates":[{"index":1,"sense_key":"once%4:02:01::","synset_id":"once.r.03"},{"index":2,"sense_key":"once%4:02:00::","synset_id":"once.r.01"},{"index":3,"sense_key":"once%4:02:02::","synset_id":"once.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"once%4:02:00::","call_ids":["semeval2015.d000.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"once%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":106},"cost":{"currency":"USD","total_usd":0.005770000000000001,"input_uncached_usd":0.0021100000000000003,"input_cached_usd":0.0,"output_usd":0.00366,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8318372920621186}
{"item_id":"semeval2015.d000.s015.t002","gold_sense_keys":["every%5:00:02:all:00"],"candidates":[{"index":1,"sense_key":"every%5:00:02:all:00","synset_id":"every.s.02"},{"index":2,"sense_key":"every%5:00:00:all:00","synset_id":"every.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"every%5:00:02:all:00","call_ids":["semeval2015.d000.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"every%5:00:02:all:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":399,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.0038550000000000004,"input_uncached_usd":0.0019950000000000002,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.870815458940342}
{"item_id":"semeval2015.d000.s015.t003","gold_sense_keys":["week%1:28:00::"],"candidates":[{"index":1,"sense_key":"week%1:28:02::","synset_id":"week.n.03"},{"index":2,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":3,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"week%1:28:00::","call_ids":["semeval2015.d000.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"week%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.004055,"input_uncached_usd":0.002075,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1264611670048907}
{"item_id":"semeval2015.d000.s015.t005","gold_sense_keys":["infusion%1:04:00::"],"candidates":[{"index":1,"sense_key":"infusion%1:04:01::","synset_id":"infusion.n.04"},{"index":2,"sense_key":"infusion%1:22:00::","synset_id":"infusion.n.02"},{"index":3,"sense_key":"infusion%1:27:00::","synset_id":"infusion.n.01"},{"index":4,"sense_key":"infusion%1:04:00::","synset_id":"infusion.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"infusion%1:04:00::","call_ids":["semeval2015.d000.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"infusion%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.0037350000000000005,"input_uncached_usd":0.0023850000000000004,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.572220291942358}
{"item_id":"semeval2015.d000.s015.t006","gold_sense_keys":["last%2:42:00::"],"candidates":[{"index":1,"sense_key":"last%2:42:01::","synset_id":"survive.v.01"},{"index":2,"sense_key":"last%2:42:00::","synset_id":"last.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"last%2:42:00::","call_ids":["semeval2015.d000.s015.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"last%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.00364,"input_uncached_usd":0.00205,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.143369499943219}
{"item_id":"semeval2015.d000.s015.t007","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":2,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":3,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"},{"index":4,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":5,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":6,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"minute%1:28:00::","call_ids":["semeval2015.d000.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.004425,"input_uncached_usd":0.002745,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3741252910112962}
{"item_id":"semeval2015.d000.s016.t000","gold_sense_keys":["reduce%2:30:00::","reduce%2:30:01::","reduce%2:40:00::"],"candidates":[{"index":1,"sense_key":"reduce%2:30:00::","synset_id":"reduce.v.01"},{"index":2,"sense_key":"reduce%2:30:02::","synset_id":"reduce.v.18"},{"index":3,"sense_key":"reduce%2:30:05::","synset_id":"shrink.v.03"},{"index":4,"sense_key":"reduce%2:30:04::","synset_id":"reduce.v.09"},{"index":5,"sense_key":"reduce%2:30:01::","synset_id":"reduce.v.11"},{"index":6,"sense_key":"reduce%2:29:00::","synset_id":"reduce.v.20"},{"index":7,"sense_key":"reduce%2:30:10::","synset_id":"boil_down.v.02"},{"index":8,"sense_key":"reduce%2:30:12::","synset_id":"reduce.v.15"},{"index":9,"sense_key":"reduce%2:30:08::","synset_id":"abridge.v.01"},{"index":10,"sense_key":"reduce%2:40:00::","synset_id":"reduce.v.08"},{"index":11,"sense_key":"reduce%2:30:11::","synset_id":"reduce.v.06"},{"index":12,"sense_key":"reduce%2:35:01::","synset_id":"reduce.v.14"},{"index":13,"sense_key":"reduce%2:40:04::","synset_id":"reduce.v.04"},{"index":14,"sense_key":"reduce%2:37:00::","synset_id":"reduce.v.05"},{"index":15,"sense_key":"reduce%2:35:00::","synset_id":"reduce.v.13"},{"index":16,"sense_key":"reduce%2:30:03::","synset_id":"deoxidize.v.01"},{"index":17,"sense_key":"reduce%2:30:09::","synset_id":"dilute.v.01"},{"index":18,"sense_key":"reduce%2:41:00::","synset_id":"reduce.v.03"},{"index":19,"sense_key":"reduce%2:41:03::","synset_id":"repress.v.01"},{"index":20,"sense_key":"reduce%2:30:06::","synset_id":"reduce.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reduce%2:30:00::","call_ids":["semeval2015.d000.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reduce%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1111,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.009065,"input_uncached_usd":0.005555,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.669796166010201}
{"item_id":"semeval2015.d000.s016.t001","gold_sense_keys":["side_effect%1:26:00::"],"candidates":[{"index":1,"sense_key":"side_effect%1:26:00::","synset_id":"side_effect.n.01"},{"index":2,"sense_key":"side_effect%1:19:00::","synset_id":"side_effect.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"side_effect%1:26:00::","call_ids":["semeval2015.d000.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"side_effect%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":395,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.003565,"input_uncached_usd":0.001975,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6511102090589702}
{"item_id":"semeval2015.d000.s016.t002","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"},{"index":2,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":397,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0036950000000000004,"input_uncached_usd":0.0019850000000000002,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6958501250483096}
{"item_id":"semeval2015.d000.s016.t006","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":2,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":3,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":4,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":5,"sense_key":"type%1:06:00::","synset_id":"type.n.06"},{"index":6,"sense_key":"type%1:09:00::","synset_id":"type.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2015.d000.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.004955,"input_uncached_usd":0.002735,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.782570083043538}
{"item_id":"semeval2015.d000.s016.t008","gold_sense_keys":["injection%1:04:01::"],"candidates":[{"index":1,"sense_key":"injection%1:27:00::","synset_id":"injection.n.02"},{"index":2,"sense_key":"injection%1:04:00::","synset_id":"injection.n.01"},{"index":3,"sense_key":"injection%1:04:01::","synset_id":"injection.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"injection%1:04:01::","call_ids":["semeval2015.d000.s016.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"injection%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.004805,"input_uncached_usd":0.002075,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6140495829749852}
{"item_id":"semeval2015.d000.s016.t010","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":2,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":3,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"},{"index":4,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d000.s016.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.004025,"input_uncached_usd":0.002435,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2577286660671234}
{"item_id":"semeval2015.d000.s017.t000","gold_sense_keys":["give%2:40:13::"],"candidates":[{"index":1,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":2,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":3,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":4,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":5,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":6,"sense_key":"give%2:29:10::","synset_id":"give.v.44"},{"index":7,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":10,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":11,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":12,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":13,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":14,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":15,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":16,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":17,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":18,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":19,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":20,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":21,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":24,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":25,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":26,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":27,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":28,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":29,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":30,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":33,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":34,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":35,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":36,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":37,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":38,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":39,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":40,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":41,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":42,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":43,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":44,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":18,"chosen_sense_key":"give%2:40:13::","call_ids":["semeval2015.d000.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":18,"predicted_sense_key":"give%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2076,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.01227,"input_uncached_usd":0.01038,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1291216670069844}
{"item_id":"semeval2015.d000.s017.t001","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"},{"index":4,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.0043,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3962334159296006}
{"item_id":"semeval2015.d000.s017.t002","gold_sense_keys":["prevent%2:41:01::"],"candidates":[{"index":1,"sense_key":"prevent%2:41:01::","synset_id":"prevent.v.01"},{"index":2,"sense_key":"prevent%2:41:00::","synset_id":"prevent.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"prevent%2:41:01::","call_ids":["semeval2015.d000.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"prevent%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":219,"reasoning_output_tokens":204},"cost":{"currency":"USD","total_usd":0.0088,"input_uncached_usd":0.00223,"input_cached_usd":0.0,"output_usd":0.00657,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.1234624170465395}
{"item_id":"semeval2015.d000.s017.t004","gold_sense_keys":["fluid%1:27:02::"],"candidates":[{"index":1,"sense_key":"fluid%1:27:02::","synset_id":"fluid.n.01"},{"index":2,"sense_key":"fluid%1:27:00::","synset_id":"fluid.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fluid%1:27:02::","call_ids":["semeval2015.d000.s017.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fluid%1:27:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":395,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017815,"input_uncached_usd":0.001975,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.737214458058588}
{"item_id":"semeval2015.d000.s017.t005","gold_sense_keys":["prevent%2:41:01::"],"candidates":[{"index":1,"sense_key":"prevent%2:41:01::","synset_id":"prevent.v.01"},{"index":2,"sense_key":"prevent%2:41:00::","synset_id":"prevent.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"prevent%2:41:01::","call_ids":["semeval2015.d000.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"prevent%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":408,"reasoning_output_tokens":392},"cost":{"currency":"USD","total_usd":0.01447,"input_uncached_usd":0.00223,"input_cached_usd":0.0,"output_usd":0.012240000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.954241250059567}
{"item_id":"semeval2015.d000.s017.t006","gold_sense_keys":["dehydration%1:26:01::"],"candidates":[{"index":1,"sense_key":"dehydration%1:26:01::","synset_id":"dehydration.n.02"},{"index":2,"sense_key":"dehydration%1:26:00::","synset_id":"dehydration.n.01"},{"index":3,"sense_key":"dehydration%1:22:00::","synset_id":"dehydration.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dehydration%1:26:01::","call_ids":["semeval2015.d000.s017.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dehydration%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":81},"cost":{"currency":"USD","total_usd":0.0050100000000000006,"input_uncached_usd":0.0021000000000000003,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4652479579672217}
{"item_id":"semeval2015.d000.s017.t008","gold_sense_keys":["give%2:40:13::"],"candidates":[{"index":1,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":2,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":3,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":4,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":7,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":8,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":9,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":10,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":11,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":12,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":13,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":14,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":15,"sense_key":"give%2:29:10::","synset_id":"give.v.44"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":18,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":19,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":20,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":21,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":22,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":23,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":24,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":25,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":26,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":27,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":28,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":29,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":30,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":31,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":32,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":33,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":34,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":35,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":36,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":37,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":38,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":39,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":40,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":41,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":42,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":43,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":44,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":22,"chosen_sense_key":"give%2:40:13::","call_ids":["semeval2015.d000.s017.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":22,"predicted_sense_key":"give%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2076,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.01287,"input_uncached_usd":0.01038,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7513622080441564}
{"item_id":"semeval2015.d000.s017.t009","gold_sense_keys":["before%4:02:03::"],"candidates":[{"index":1,"sense_key":"before%4:02:03::","synset_id":"earlier.r.01"},{"index":2,"sense_key":"before%4:02:01::","synset_id":"ahead.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"before%4:02:03::","call_ids":["semeval2015.d000.s017.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"before%4:02:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.004025,"input_uncached_usd":0.002075,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":71.91218874999322}
{"item_id":"semeval2015.d000.s017.t010","gold_sense_keys":["after%4:02:01::"],"candidates":[{"index":1,"sense_key":"after%4:02:01::","synset_id":"subsequently.r.01"},{"index":2,"sense_key":"after%4:02:00::","synset_id":"after.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"after%4:02:01::","call_ids":["semeval2015.d000.s017.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"after%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.003705,"input_uncached_usd":0.002085,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5824923330219463}
{"item_id":"semeval2015.d000.s017.t011","gold_sense_keys":["dose%1:06:00::"],"candidates":[{"index":1,"sense_key":"dose%1:06:01::","synset_id":"acid.n.02"},{"index":2,"sense_key":"dose%1:23:00::","synset_id":"dose.n.02"},{"index":3,"sense_key":"dose%1:26:00::","synset_id":"venereal_disease.n.01"},{"index":4,"sense_key":"dose%1:06:00::","synset_id":"dose.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"dose%1:06:00::","call_ids":["semeval2015.d000.s017.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"dose%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":450,"reasoning_output_tokens":434},"cost":{"currency":"USD","total_usd":0.015985,"input_uncached_usd":0.0024850000000000002,"input_cached_usd":0.0,"output_usd":0.0135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.490520874969661}
{"item_id":"semeval2015.d000.s018.t000","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":2,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":3,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"},{"index":4,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d000.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.004605,"input_uncached_usd":0.002565,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.797369125066325}
{"item_id":"semeval2015.d000.s018.t001","gold_sense_keys":["delay%2:30:00::"],"candidates":[{"index":1,"sense_key":"delay%2:42:00::","synset_id":"delay.v.02"},{"index":2,"sense_key":"delay%2:30:01::","synset_id":"stay.v.06"},{"index":3,"sense_key":"delay%2:30:00::","synset_id":"delay.v.01"},{"index":4,"sense_key":"delay%2:30:05::","synset_id":"check.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"delay%2:30:00::","call_ids":["semeval2015.d000.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"delay%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018385,"input_uncached_usd":0.0025450000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.779806584003381}
{"item_id":"semeval2015.d000.s018.t002","gold_sense_keys":["discontinue%2:42:00::"],"candidates":[{"index":1,"sense_key":"discontinue%2:42:00::","synset_id":"discontinue.v.01"},{"index":2,"sense_key":"discontinue%2:30:00::","synset_id":"break.v.10"},{"index":3,"sense_key":"discontinue%2:42:01::","synset_id":"discontinue.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discontinue%2:42:00::","call_ids":["semeval2015.d000.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discontinue%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":466,"cached_input_tokens":0,"output_tokens":328,"reasoning_output_tokens":313},"cost":{"currency":"USD","total_usd":0.012170000000000002,"input_uncached_usd":0.00233,"input_cached_usd":0.0,"output_usd":0.00984,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.908080708933994}
{"item_id":"semeval2015.d000.s018.t003","gold_sense_keys":["dose%1:06:00::"],"candidates":[{"index":1,"sense_key":"dose%1:23:00::","synset_id":"dose.n.02"},{"index":2,"sense_key":"dose%1:26:00::","synset_id":"venereal_disease.n.01"},{"index":3,"sense_key":"dose%1:06:00::","synset_id":"dose.n.01"},{"index":4,"sense_key":"dose%1:06:01::","synset_id":"acid.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"dose%1:06:00::","call_ids":["semeval2015.d000.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"dose%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":192,"reasoning_output_tokens":176},"cost":{"currency":"USD","total_usd":0.0083,"input_uncached_usd":0.00254,"input_cached_usd":0.0,"output_usd":0.00576,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6270892079919577}
{"item_id":"semeval2015.d000.s018.t004","gold_sense_keys":["reduce%2:30:00::"],"candidates":[{"index":1,"sense_key":"reduce%2:40:00::","synset_id":"reduce.v.08"},{"index":2,"sense_key":"reduce%2:30:11::","synset_id":"reduce.v.06"},{"index":3,"sense_key":"reduce%2:30:10::","synset_id":"boil_down.v.02"},{"index":4,"sense_key":"reduce%2:30:06::","synset_id":"reduce.v.02"},{"index":5,"sense_key":"reduce%2:35:00::","synset_id":"reduce.v.13"},{"index":6,"sense_key":"reduce%2:30:09::","synset_id":"dilute.v.01"},{"index":7,"sense_key":"reduce%2:41:00::","synset_id":"reduce.v.03"},{"index":8,"sense_key":"reduce%2:35:01::","synset_id":"reduce.v.14"},{"index":9,"sense_key":"reduce%2:30:01::","synset_id":"reduce.v.11"},{"index":10,"sense_key":"reduce%2:37:00::","synset_id":"reduce.v.05"},{"index":11,"sense_key":"reduce%2:30:12::","synset_id":"reduce.v.15"},{"index":12,"sense_key":"reduce%2:30:00::","synset_id":"reduce.v.01"},{"index":13,"sense_key":"reduce%2:30:02::","synset_id":"reduce.v.18"},{"index":14,"sense_key":"reduce%2:40:04::","synset_id":"reduce.v.04"},{"index":15,"sense_key":"reduce%2:30:03::","synset_id":"deoxidize.v.01"},{"index":16,"sense_key":"reduce%2:30:05::","synset_id":"shrink.v.03"},{"index":17,"sense_key":"reduce%2:41:03::","synset_id":"repress.v.01"},{"index":18,"sense_key":"reduce%2:29:00::","synset_id":"reduce.v.20"},{"index":19,"sense_key":"reduce%2:30:08::","synset_id":"abridge.v.01"},{"index":20,"sense_key":"reduce%2:30:04::","synset_id":"reduce.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"reduce%2:30:00::","call_ids":["semeval2015.d000.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"reduce%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1138,"cached_input_tokens":0,"output_tokens":461,"reasoning_output_tokens":445},"cost":{"currency":"USD","total_usd":0.019520000000000003,"input_uncached_usd":0.005690000000000001,"input_cached_usd":0.0,"output_usd":0.01383,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":81.84302558307536}
{"item_id":"semeval2015.d000.s018.t005","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":40,"reasoning_output_tokens":25},"cost":{"currency":"USD","total_usd":0.0033200000000000005,"input_uncached_usd":0.0021200000000000004,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.392224375042133}
{"item_id":"semeval2015.d000.s018.t006","gold_sense_keys":["blood_count%1:23:00::"],"candidates":[{"index":1,"sense_key":"blood_count%1:04:00::","synset_id":"blood_count.n.02"},{"index":2,"sense_key":"blood_count%1:23:00::","synset_id":"blood_count.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"blood_count%1:23:00::","call_ids":["semeval2015.d000.s018.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"blood_count%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.003935,"input_uncached_usd":0.0020150000000000003,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.887535999994725}
{"item_id":"semeval2015.d000.s018.t007","gold_sense_keys":["abnormal%3:00:00::"],"candidates":[{"index":1,"sense_key":"abnormal%3:00:03::","synset_id":"abnormal.a.02"},{"index":2,"sense_key":"abnormal%3:00:00::","synset_id":"abnormal.a.01"},{"index":3,"sense_key":"abnormal%5:00:00:immoderate:00","synset_id":"abnormal.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"abnormal%3:00:00::","call_ids":["semeval2015.d000.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"abnormal%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.00486,"input_uncached_usd":0.00237,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1505967499688268}
{"item_id":"semeval2015.d000.s018.t008","gold_sense_keys":["have%2:30:01::","have%2:39:06::"],"candidates":[{"index":1,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":2,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":3,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":4,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":5,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":6,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":9,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":10,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":13,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":14,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":15,"sense_key":"have%2:35:00::","synset_id":"take.v.35"},{"index":16,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":17,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":18,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":19,"sense_key":"have%2:42:12::","synset_id":"have.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"have%2:29:05::","call_ids":["semeval2015.d000.s018.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"have%2:29:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1173,"cached_input_tokens":0,"output_tokens":293,"reasoning_output_tokens":277},"cost":{"currency":"USD","total_usd":0.014655,"input_uncached_usd":0.005865,"input_cached_usd":0.0,"output_usd":0.008790000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.071243582991883}
{"item_id":"semeval2015.d000.s018.t009","gold_sense_keys":["certain%5:00:00:definite:00"],"candidates":[{"index":1,"sense_key":"certain%5:00:00:definite:00","synset_id":"certain.s.01"},{"index":2,"sense_key":"certain%5:00:00:careful:00","synset_id":"certain.s.03"},{"index":3,"sense_key":"certain%5:00:00:reliable:00","synset_id":"certain.s.02"},{"index":4,"sense_key":"certain%3:00:02::","synset_id":"certain.a.02"},{"index":5,"sense_key":"certain%3:00:05::","synset_id":"sealed.a.01"},{"index":6,"sense_key":"certain%3:00:01::","synset_id":"certain.a.03"},{"index":7,"sense_key":"certain%3:00:03::","synset_id":"certain.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"certain%5:00:00:definite:00","call_ids":["semeval2015.d000.s018.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"certain%5:00:00:definite:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.005125,"input_uncached_usd":0.003295,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7396470409585163}
{"item_id":"semeval2015.d000.s018.t010","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":2,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%3:00:00::","synset_id":"other.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"other%3:00:00::","call_ids":["semeval2015.d000.s018.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.004775000000000001,"input_uncached_usd":0.0025550000000000004,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6235882920445874}
{"item_id":"semeval2015.d000.s018.t011","gold_sense_keys":["side_effect%1:26:00::"],"candidates":[{"index":1,"sense_key":"side_effect%1:19:00::","synset_id":"side_effect.n.02"},{"index":2,"sense_key":"side_effect%1:26:00::","synset_id":"side_effect.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"side_effect%1:26:00::","call_ids":["semeval2015.d000.s018.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"side_effect%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.0035800000000000003,"input_uncached_usd":0.0021100000000000003,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1892236249987036}
{"item_id":"semeval2015.d000.s019.t002","gold_sense_keys":["use%1:04:00::"],"candidates":[{"index":1,"sense_key":"use%1:04:01::","synset_id":"habit.n.02"},{"index":2,"sense_key":"use%1:22:00::","synset_id":"consumption.n.03"},{"index":3,"sense_key":"use%1:07:00::","synset_id":"function.n.02"},{"index":4,"sense_key":"use%1:04:02::","synset_id":"manipulation.n.01"},{"index":5,"sense_key":"use%1:07:03::","synset_id":"use.n.07"},{"index":6,"sense_key":"use%1:04:00::","synset_id":"use.n.01"},{"index":7,"sense_key":"use%1:07:02::","synset_id":"use.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"use%1:04:00::","call_ids":["semeval2015.d000.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"use%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":81},"cost":{"currency":"USD","total_usd":0.00635,"input_uncached_usd":0.0034400000000000003,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.217466957983561}
{"item_id":"semeval2015.d000.s019.t003","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.0034400000000000003,"input_uncached_usd":0.0020900000000000003,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.945906334090978}
{"item_id":"semeval2015.d000.s019.t004","gold_sense_keys":["moderate%3:00:00::","moderate%5:00:00:mild:00"],"candidates":[{"index":1,"sense_key":"moderate%5:00:00:mild:00","synset_id":"moderate.s.01"},{"index":2,"sense_key":"moderate%3:00:00::","synset_id":"moderate.a.01"},{"index":3,"sense_key":"moderate%5:00:00:temperate:00","synset_id":"moderate.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"moderate%3:00:00::","call_ids":["semeval2015.d000.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"moderate%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":2547,"reasoning_output_tokens":2507},"cost":{"currency":"USD","total_usd":0.07871,"input_uncached_usd":0.0023000000000000004,"input_cached_usd":0.0,"output_usd":0.07641,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":57.034085375024006}
{"item_id":"semeval2015.d000.s019.t005","gold_sense_keys":["severe%5:00:00:critical:03","severe%5:00:00:intense:00","severe%5:00:01:bad:00"],"candidates":[{"index":1,"sense_key":"severe%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":2,"sense_key":"severe%5:00:00:intense:00","synset_id":"severe.s.01"},{"index":3,"sense_key":"severe%5:00:00:plain:01","synset_id":"austere.s.01"},{"index":4,"sense_key":"severe%5:00:00:critical:03","synset_id":"dangerous.s.01"},{"index":5,"sense_key":"severe%5:00:01:bad:00","synset_id":"severe.s.06"},{"index":6,"sense_key":"severe%5:00:02:nonindulgent:00","synset_id":"severe.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"severe%5:00:00:intense:00","call_ids":["semeval2015.d000.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"severe%5:00:00:intense:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":304,"reasoning_output_tokens":288},"cost":{"currency":"USD","total_usd":0.012235,"input_uncached_usd":0.003115,"input_cached_usd":0.0,"output_usd":0.00912,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.407303999993019}
{"item_id":"semeval2015.d000.s019.t007","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:26:00::","call_ids":["semeval2015.d000.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0181,"input_uncached_usd":0.0022600000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.326836500084028}
{"item_id":"semeval2015.d000.s020.t000","gold_sense_keys":["more%3:00:01::"],"candidates":[{"index":1,"sense_key":"more%3:00:02::","synset_id":"more.a.02"},{"index":2,"sense_key":"more%3:00:01::","synset_id":"more.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"more%3:00:01::","call_ids":["semeval2015.d000.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"more%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.004,"input_uncached_usd":0.00205,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6175010000588372}
{"item_id":"semeval2015.d000.s020.t001","gold_sense_keys":["information%1:09:00::"],"candidates":[{"index":1,"sense_key":"information%1:10:02::","synset_id":"information.n.03"},{"index":2,"sense_key":"information%1:10:00::","synset_id":"information.n.01"},{"index":3,"sense_key":"information%1:09:00::","synset_id":"information.n.02"},{"index":4,"sense_key":"information%1:07:00::","synset_id":"information.n.05"},{"index":5,"sense_key":"information%1:14:00::","synset_id":"data.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"information%1:10:00::","call_ids":["semeval2015.d000.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"information%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":463,"cached_input_tokens":0,"output_tokens":895,"reasoning_output_tokens":873},"cost":{"currency":"USD","total_usd":0.029165000000000003,"input_uncached_usd":0.0023150000000000002,"input_cached_usd":0.0,"output_usd":0.026850000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.861720207962207}
{"item_id":"semeval2015.d000.s020.t002","gold_sense_keys":["see%2:39:13::"],"candidates":[{"index":1,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":2,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":3,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":4,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":5,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":6,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":7,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":8,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":9,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":10,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":11,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":12,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":13,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":14,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":15,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":16,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":17,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":18,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":19,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":20,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":21,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":22,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"},{"index":23,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":24,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"see%2:39:13::","call_ids":["semeval2015.d000.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"see%2:39:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1453,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.038645,"input_uncached_usd":0.007265000000000001,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":19.715480917016976}
{"item_id":"semeval2015.d000.s020.t004","gold_sense_keys":["product%1:06:01::"],"candidates":[{"index":1,"sense_key":"product%1:06:01::","synset_id":"merchandise.n.01"},{"index":2,"sense_key":"product%1:27:00::","synset_id":"product.n.04"},{"index":3,"sense_key":"product%1:09:00::","synset_id":"product.n.03"},{"index":4,"sense_key":"product%1:14:00::","synset_id":"intersection.n.04"},{"index":5,"sense_key":"product%1:06:00::","synset_id":"product.n.02"},{"index":6,"sense_key":"product%1:19:00::","synset_id":"product.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"product%1:06:01::","call_ids":["semeval2015.d000.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"product%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":1564,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.04986500000000001,"input_uncached_usd":0.002945,"input_cached_usd":0.0,"output_usd":0.04692,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":32.05915912508499}
{"item_id":"semeval2015.d000.s020.t006","gold_sense_keys":["part%1:09:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"},{"index":2,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":3,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":4,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":5,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":6,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":7,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":8,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":9,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":10,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":11,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":12,"sense_key":"part%1:09:01::","synset_id":"character.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"part%1:24:00::","call_ids":["semeval2015.d000.s020.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"part%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":902,"cached_input_tokens":0,"output_tokens":1564,"reasoning_output_tokens":1535},"cost":{"currency":"USD","total_usd":0.051430000000000003,"input_uncached_usd":0.00451,"input_cached_usd":0.0,"output_usd":0.04692,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":27.419005749979988}
{"item_id":"semeval2015.d000.s022.t000","gold_sense_keys":["active%3:00:07::"],"candidates":[{"index":1,"sense_key":"active%3:00:01::","synset_id":"active.a.05"},{"index":2,"sense_key":"active%5:00:00:existent:00","synset_id":"active.s.03"},{"index":3,"sense_key":"active%3:00:04::","synset_id":"active.a.13"},{"index":4,"sense_key":"active%3:00:03::","synset_id":"active.a.03"},{"index":5,"sense_key":"active%5:00:00:operational:00","synset_id":"active.s.01"},{"index":6,"sense_key":"active%3:00:06::","synset_id":"active.a.07"},{"index":7,"sense_key":"active%5:00:00:involved:00","synset_id":"active.s.02"},{"index":8,"sense_key":"active%3:00:00::","synset_id":"active.a.11"},{"index":9,"sense_key":"active%3:00:10::","synset_id":"active.a.09"},{"index":10,"sense_key":"active%3:00:09::","synset_id":"active.a.10"},{"index":11,"sense_key":"active%3:00:02::","synset_id":"active.a.01"},{"index":12,"sense_key":"active%3:00:08::","synset_id":"active.a.14"},{"index":13,"sense_key":"active%3:00:05::","synset_id":"active.a.12"},{"index":14,"sense_key":"active%3:00:07::","synset_id":"active.a.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"active%3:00:07::","call_ids":["semeval2015.d000.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"active%3:00:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":889,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.006905,"input_uncached_usd":0.004445,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8560607919935137}
{"item_id":"semeval2015.d000.s022.t001","gold_sense_keys":["substance%1:03:01::"],"candidates":[{"index":1,"sense_key":"substance%1:03:01::","synset_id":"substance.n.07"},{"index":2,"sense_key":"substance%1:09:01::","synset_id":"kernel.n.03"},{"index":3,"sense_key":"substance%1:03:00::","synset_id":"substance.n.01"},{"index":4,"sense_key":"substance%1:21:00::","synset_id":"means.n.03"},{"index":5,"sense_key":"substance%1:09:00::","synset_id":"meaning.n.02"},{"index":6,"sense_key":"substance%1:10:00::","synset_id":"message.n.02"},{"index":7,"sense_key":"substance%1:27:00::","synset_id":"substance.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"substance%1:03:01::","call_ids":["semeval2015.d000.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"substance%1:03:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":632,"cached_input_tokens":0,"output_tokens":894,"reasoning_output_tokens":873},"cost":{"currency":"USD","total_usd":0.02998,"input_uncached_usd":0.00316,"input_cached_usd":0.0,"output_usd":0.02682,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":17.675052125006914}
{"item_id":"semeval2015.d000.s022.t003","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"},{"index":2,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":3,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":4,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.004350000000000001,"input_uncached_usd":0.0025800000000000003,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9031601670430973}
{"item_id":"semeval2015.d000.s022.t004","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.00477,"input_uncached_usd":0.0025800000000000003,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9168419999768957}
{"item_id":"semeval2015.d000.s022.t005","gold_sense_keys":["kill%2:35:00::","kill%2:35:01::","kill%2:35:02::"],"candidates":[{"index":1,"sense_key":"kill%2:35:02::","synset_id":"kill.v.09"},{"index":2,"sense_key":"kill%2:30:03::","synset_id":"kill.v.14"},{"index":3,"sense_key":"kill%2:42:00::","synset_id":"kill.v.04"},{"index":4,"sense_key":"kill%2:35:00::","synset_id":"kill.v.01"},{"index":5,"sense_key":"kill%2:30:02::","synset_id":"kill.v.15"},{"index":6,"sense_key":"kill%2:30:04::","synset_id":"kill.v.13"},{"index":7,"sense_key":"kill%2:30:00::","synset_id":"kill.v.12"},{"index":8,"sense_key":"kill%2:39:00::","synset_id":"kill.v.05"},{"index":9,"sense_key":"kill%2:30:08::","synset_id":"stamp_out.v.01"},{"index":10,"sense_key":"kill%2:35:05::","synset_id":"kill.v.08"},{"index":11,"sense_key":"kill%2:35:01::","synset_id":"kill.v.10"},{"index":12,"sense_key":"kill%2:41:01::","synset_id":"kill.v.02"},{"index":13,"sense_key":"kill%2:35:06::","synset_id":"kill.v.07"},{"index":14,"sense_key":"kill%2:37:00::","synset_id":"kill.v.06"},{"index":15,"sense_key":"kill%2:34:00::","synset_id":"toss_off.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"kill%2:35:02::","call_ids":["semeval2015.d000.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"kill%2:35:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":919,"cached_input_tokens":0,"output_tokens":2081,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.067025,"input_uncached_usd":0.004595,"input_cached_usd":0.0,"output_usd":0.06243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":46.201950415968895}
{"item_id":"semeval2015.d000.s022.t006","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":3,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":4,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":7,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"cell%1:03:00::","call_ids":["semeval2015.d000.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00503,"input_uncached_usd":0.0030800000000000003,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8612885419279337}
{"item_id":"semeval2015.d000.s022.t007","gold_sense_keys":["divide%2:35:00::"],"candidates":[{"index":1,"sense_key":"divide%2:42:00::","synset_id":"separate.v.01"},{"index":2,"sense_key":"divide%2:35:01::","synset_id":"separate.v.02"},{"index":3,"sense_key":"divide%2:38:00::","synset_id":"separate.v.07"},{"index":4,"sense_key":"divide%2:41:00::","synset_id":"divide.v.01"},{"index":5,"sense_key":"divide%2:31:00::","synset_id":"divide.v.02"},{"index":6,"sense_key":"divide%2:35:00::","synset_id":"separate.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"divide%2:38:00::","call_ids":["semeval2015.d000.s022.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"divide%2:38:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":590,"cached_input_tokens":0,"output_tokens":3341,"reasoning_output_tokens":3289},"cost":{"currency":"USD","total_usd":0.10318000000000001,"input_uncached_usd":0.0029500000000000004,"input_cached_usd":0.0,"output_usd":0.10023,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":50.32401129195932}
{"item_id":"semeval2015.d000.s023.t000","gold_sense_keys":["body%1:08:00::"],"candidates":[{"index":1,"sense_key":"body%1:14:02::","synset_id":"body.n.06"},{"index":2,"sense_key":"body%1:07:00::","synset_id":"consistency.n.01"},{"index":3,"sense_key":"body%1:06:01::","synset_id":"soundbox.n.01"},{"index":4,"sense_key":"body%1:17:00::","synset_id":"body.n.04"},{"index":5,"sense_key":"body%1:10:00::","synset_id":"body.n.08"},{"index":6,"sense_key":"body%1:08:02::","synset_id":"body.n.03"},{"index":7,"sense_key":"body%1:06:00::","synset_id":"body.n.11"},{"index":8,"sense_key":"body%1:08:00::","synset_id":"body.n.01"},{"index":9,"sense_key":"body%1:08:01::","synset_id":"torso.n.01"},{"index":10,"sense_key":"body%1:07:01::","synset_id":"body.n.09"},{"index":11,"sense_key":"body%1:14:00::","synset_id":"body.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"body%1:08:00::","call_ids":["semeval2015.d000.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"body%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.005560000000000001,"input_uncached_usd":0.0037900000000000004,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4088062499649823}
{"item_id":"semeval2015.d000.s023.t001","gold_sense_keys":["convert%2:30:03::"],"candidates":[{"index":1,"sense_key":"convert%2:33:01::","synset_id":"convert.v.07"},{"index":2,"sense_key":"convert%2:30:07::","synset_id":"convert.v.05"},{"index":3,"sense_key":"convert%2:32:00::","synset_id":"convert.v.09"},{"index":4,"sense_key":"convert%2:30:04::","synset_id":"convert.v.11"},{"index":5,"sense_key":"convert%2:33:02::","synset_id":"convert.v.06"},{"index":6,"sense_key":"convert%2:30:09::","synset_id":"commute.v.04"},{"index":7,"sense_key":"convert%2:30:03::","synset_id":"convert.v.02"},{"index":8,"sense_key":"convert%2:30:00::","synset_id":"convert.v.01"},{"index":9,"sense_key":"convert%2:30:01::","synset_id":"convert.v.03"},{"index":10,"sense_key":"convert%2:30:02::","synset_id":"change.v.06"},{"index":11,"sense_key":"convert%2:33:00::","synset_id":"convert.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"convert%2:30:03::","call_ids":["semeval2015.d000.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"convert%2:30:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":744,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0351,"input_uncached_usd":0.00372,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":18.46930204203818}
{"item_id":"semeval2015.d000.s023.t002","gold_sense_keys":["active%3:00:07::"],"candidates":[{"index":1,"sense_key":"active%3:00:06::","synset_id":"active.a.07"},{"index":2,"sense_key":"active%5:00:00:operational:00","synset_id":"active.s.01"},{"index":3,"sense_key":"active%3:00:04::","synset_id":"active.a.13"},{"index":4,"sense_key":"active%3:00:02::","synset_id":"active.a.01"},{"index":5,"sense_key":"active%3:00:08::","synset_id":"active.a.14"},{"index":6,"sense_key":"active%3:00:03::","synset_id":"active.a.03"},{"index":7,"sense_key":"active%3:00:09::","synset_id":"active.a.10"},{"index":8,"sense_key":"active%5:00:00:existent:00","synset_id":"active.s.03"},{"index":9,"sense_key":"active%3:00:10::","synset_id":"active.a.09"},{"index":10,"sense_key":"active%3:00:01::","synset_id":"active.a.05"},{"index":11,"sense_key":"active%3:00:05::","synset_id":"active.a.12"},{"index":12,"sense_key":"active%5:00:00:involved:00","synset_id":"active.s.02"},{"index":13,"sense_key":"active%3:00:07::","synset_id":"active.a.06"},{"index":14,"sense_key":"active%3:00:00::","synset_id":"active.a.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"active%3:00:07::","call_ids":["semeval2015.d000.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"active%3:00:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":856,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.006529999999999999,"input_uncached_usd":0.00428,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1874124170280993}
{"item_id":"semeval2015.d000.s023.t003","gold_sense_keys":["form%1:07:03::"],"candidates":[{"index":1,"sense_key":"form%1:09:01::","synset_id":"kind.n.01"},{"index":2,"sense_key":"form%1:07:00::","synset_id":"form.n.07"},{"index":3,"sense_key":"form%1:07:03::","synset_id":"form.n.11"},{"index":4,"sense_key":"form%1:10:02::","synset_id":"form.n.10"},{"index":5,"sense_key":"form%1:08:00::","synset_id":"human_body.n.01"},{"index":6,"sense_key":"form%1:14:02::","synset_id":"form.n.09"},{"index":7,"sense_key":"form%1:06:01::","synset_id":"form.n.16"},{"index":8,"sense_key":"form%1:03:00::","synset_id":"shape.n.02"},{"index":9,"sense_key":"form%1:26:00::","synset_id":"phase.n.02"},{"index":10,"sense_key":"form%1:10:00::","synset_id":"form.n.01"},{"index":11,"sense_key":"form%1:07:02::","synset_id":"form.n.14"},{"index":12,"sense_key":"form%1:07:01::","synset_id":"shape.n.01"},{"index":13,"sense_key":"form%1:06:00::","synset_id":"mannequin.n.02"},{"index":14,"sense_key":"form%1:14:00::","synset_id":"class.n.02"},{"index":15,"sense_key":"form%1:09:00::","synset_id":"form.n.03"},{"index":16,"sense_key":"form%1:10:01::","synset_id":"form.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"form%1:07:03::","call_ids":["semeval2015.d000.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"form%1:07:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1043,"cached_input_tokens":0,"output_tokens":338,"reasoning_output_tokens":322},"cost":{"currency":"USD","total_usd":0.015355,"input_uncached_usd":0.005215,"input_cached_usd":0.0,"output_usd":0.01014,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.123189916019328}
{"item_id":"semeval2015.d000.s023.t004","gold_sense_keys":["block%2:41:01::"],"candidates":[{"index":1,"sense_key":"block%2:35:00::","synset_id":"barricade.v.01"},{"index":2,"sense_key":"block%2:35:06::","synset_id":"block.v.11"},{"index":3,"sense_key":"block%2:39:05::","synset_id":"block.v.06"},{"index":4,"sense_key":"block%2:35:03::","synset_id":"block.v.08"},{"index":5,"sense_key":"block%2:30:01::","synset_id":"block.v.14"},{"index":6,"sense_key":"block%2:41:01::","synset_id":"stop.v.03"},{"index":7,"sense_key":"block%2:39:00::","synset_id":"obstruct.v.03"},{"index":8,"sense_key":"block%2:35:12::","synset_id":"obstruct.v.02"},{"index":9,"sense_key":"block%2:31:00::","synset_id":"forget.v.02"},{"index":10,"sense_key":"block%2:41:00::","synset_id":"obstruct.v.01"},{"index":11,"sense_key":"block%2:41:03::","synset_id":"block.v.05"},{"index":12,"sense_key":"block%2:35:02::","synset_id":"stuff.v.03"},{"index":13,"sense_key":"block%2:30:00::","synset_id":"block.v.15"},{"index":14,"sense_key":"block%2:30:02::","synset_id":"freeze.v.08"},{"index":15,"sense_key":"block%2:33:00::","synset_id":"parry.v.01"},{"index":16,"sense_key":"block%2:32:00::","synset_id":"jam.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"block%2:41:01::","call_ids":["semeval2015.d000.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"block%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":998,"cached_input_tokens":0,"output_tokens":184,"reasoning_output_tokens":168},"cost":{"currency":"USD","total_usd":0.010510000000000002,"input_uncached_usd":0.0049900000000000005,"input_cached_usd":0.0,"output_usd":0.00552,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.551557375001721}
{"item_id":"semeval2015.d000.s023.t007","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"},{"index":2,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"},{"index":3,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"},{"index":4,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"},{"index":5,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"involved%3:00:00::","call_ids":["semeval2015.d000.s023.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.004365,"input_uncached_usd":0.002745,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4214029170107096}
{"item_id":"semeval2015.d000.s023.t008","gold_sense_keys":["produce%2:36:02::","produce%2:36:03::"],"candidates":[{"index":1,"sense_key":"produce%2:29:00::","synset_id":"grow.v.08"},{"index":2,"sense_key":"produce%2:39:01::","synset_id":"produce.v.04"},{"index":3,"sense_key":"produce%2:36:00::","synset_id":"produce.v.02"},{"index":4,"sense_key":"produce%2:39:00::","synset_id":"produce.v.06"},{"index":5,"sense_key":"produce%2:36:03::","synset_id":"produce.v.03"},{"index":6,"sense_key":"produce%2:36:05::","synset_id":"grow.v.07"},{"index":7,"sense_key":"produce%2:36:02::","synset_id":"produce.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"produce%2:36:00::","call_ids":["semeval2015.d000.s023.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"produce%2:36:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":651,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019065000000000002,"input_uncached_usd":0.003255,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.657120208954439}
{"item_id":"semeval2015.d000.s023.t010","gold_sense_keys":["building_block%1:17:00::"],"candidates":[{"index":1,"sense_key":"building_block%1:06:00::","synset_id":"building_block.n.02"},{"index":2,"sense_key":"building_block%1:17:00::","synset_id":"unit.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"building_block%1:17:00::","call_ids":["semeval2015.d000.s023.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"building_block%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":380,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0035800000000000003,"input_uncached_usd":0.0019000000000000002,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.695423708995804}
{"item_id":"semeval2015.d000.s023.t013","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":2,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":3,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"},{"index":4,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genetic%3:01:02::","call_ids":["semeval2015.d000.s023.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.005985000000000001,"input_uncached_usd":0.002445,"input_cached_usd":0.0,"output_usd":0.00354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.847633791039698}
{"item_id":"semeval2015.d000.s023.t014","gold_sense_keys":["material%1:27:00::"],"candidates":[{"index":1,"sense_key":"material%1:06:01::","synset_id":"material.n.04"},{"index":2,"sense_key":"material%1:27:00::","synset_id":"material.n.01"},{"index":3,"sense_key":"material%1:10:00::","synset_id":"material.n.02"},{"index":4,"sense_key":"material%1:06:00::","synset_id":"fabric.n.01"},{"index":5,"sense_key":"material%1:18:00::","synset_id":"material.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"material%1:27:00::","call_ids":["semeval2015.d000.s023.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"material%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":550,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":135},"cost":{"currency":"USD","total_usd":0.00728,"input_uncached_usd":0.0027500000000000003,"input_cached_usd":0.0,"output_usd":0.00453,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6655414580600336}
{"item_id":"semeval2015.d000.s023.t015","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":4,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":5,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":6,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":7,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["semeval2015.d000.s023.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.00438,"input_uncached_usd":0.0029100000000000003,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5194051660364494}
{"item_id":"semeval2015.d000.s024.t000","gold_sense_keys":["study%2:31:02::"],"candidates":[{"index":1,"sense_key":"study%2:31:03::","synset_id":"study.v.02"},{"index":2,"sense_key":"study%2:31:04::","synset_id":"study.v.06"},{"index":3,"sense_key":"study%2:31:02::","synset_id":"analyze.v.01"},{"index":4,"sense_key":"study%2:31:01::","synset_id":"study.v.05"},{"index":5,"sense_key":"study%2:31:00::","synset_id":"learn.v.04"},{"index":6,"sense_key":"study%2:39:00::","synset_id":"study.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%2:31:02::","call_ids":["semeval2015.d000.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.006,"input_uncached_usd":0.0029400000000000003,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.891359749948606}
{"item_id":"semeval2015.d000.s025.t000","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d000.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.00448,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6054023340111598}
{"item_id":"semeval2015.d000.s025.t004","gold_sense_keys":["study%2:31:02::"],"candidates":[{"index":1,"sense_key":"study%2:31:00::","synset_id":"learn.v.04"},{"index":2,"sense_key":"study%2:31:02::","synset_id":"analyze.v.01"},{"index":3,"sense_key":"study%2:31:03::","synset_id":"study.v.02"},{"index":4,"sense_key":"study%2:31:01::","synset_id":"study.v.05"},{"index":5,"sense_key":"study%2:31:04::","synset_id":"study.v.06"},{"index":6,"sense_key":"study%2:39:00::","synset_id":"study.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"study%2:31:02::","call_ids":["semeval2015.d000.s025.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"study%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":146},"cost":{"currency":"USD","total_usd":0.00783,"input_uncached_usd":0.0029700000000000004,"input_cached_usd":0.0,"output_usd":0.00486,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6439653339330107}
{"item_id":"semeval2015.d000.s025.t005","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":2,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"},{"index":3,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["semeval2015.d000.s025.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.0038250000000000003,"input_uncached_usd":0.0023250000000000002,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.714859542087652}
{"item_id":"semeval2015.d000.s025.t006","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":2,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":3,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":4,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":5,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":6,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":7,"sense_key":"study%1:10:01::","synset_id":"study.n.10"},{"index":8,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":9,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":10,"sense_key":"study%1:18:00::","synset_id":"study.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d000.s025.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":213,"reasoning_output_tokens":197},"cost":{"currency":"USD","total_usd":0.010075,"input_uncached_usd":0.0036850000000000003,"input_cached_usd":0.0,"output_usd":0.00639,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.940452874987386}
{"item_id":"semeval2015.d000.s025.t007","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"},{"index":2,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s025.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":43,"reasoning_output_tokens":27},"cost":{"currency":"USD","total_usd":0.00334,"input_uncached_usd":0.00205,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2083042920567095}
{"item_id":"semeval2015.d000.s025.t012","gold_sense_keys":["before%4:02:03::"],"candidates":[{"index":1,"sense_key":"before%4:02:01::","synset_id":"ahead.r.01"},{"index":2,"sense_key":"before%4:02:03::","synset_id":"earlier.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"before%4:02:03::","call_ids":["semeval2015.d000.s025.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"before%4:02:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.00343,"input_uncached_usd":0.00205,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.985419958946295}
{"item_id":"semeval2015.d000.s026.t000","gold_sense_keys":["effect%1:26:00::"],"candidates":[{"index":1,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":2,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":3,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"},{"index":4,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"},{"index":5,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"},{"index":6,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"effect%1:19:00::","call_ids":["semeval2015.d000.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"effect%1:19:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01893,"input_uncached_usd":0.0030900000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.958918584045023}
{"item_id":"semeval2015.d000.s026.t002","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":2,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":3,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":4,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"compare%2:31:00::","call_ids":["semeval2015.d000.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004675,"input_uncached_usd":0.002635,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.256572625017725}
{"item_id":"semeval2015.d000.s027.t000","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"},{"index":4,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d000.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004635,"input_uncached_usd":0.0028650000000000004,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2128547090105712}
{"item_id":"semeval2015.d000.s027.t001","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":2,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":3,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":4,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":5,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":6,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":7,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cell%1:03:00::","call_ids":["semeval2015.d000.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":79},"cost":{"currency":"USD","total_usd":0.006165,"input_uncached_usd":0.0033450000000000003,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9559697919758037}
{"item_id":"semeval2015.d000.s027.t003","gold_sense_keys":["effect%1:26:00::"],"candidates":[{"index":1,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":2,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"},{"index":3,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":4,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"},{"index":5,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"},{"index":6,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"effect%1:19:00::","call_ids":["semeval2015.d000.s027.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"effect%1:19:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":657,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019125,"input_uncached_usd":0.003285,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":16.837287583039142}
{"item_id":"semeval2015.d000.s027.t004","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":2,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"},{"index":3,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":4,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"compare%2:31:00::","call_ids":["semeval2015.d000.s027.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":566,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.00553,"input_uncached_usd":0.00283,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7847085410030559}
{"item_id":"semeval2015.d000.s027.t006","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"},{"index":4,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s027.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":28},"cost":{"currency":"USD","total_usd":0.0041600000000000005,"input_uncached_usd":0.00284,"input_cached_usd":0.0,"output_usd":0.00132,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.188468666980043}
{"item_id":"semeval2015.d000.s027.t007","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:01::","synset_id":"study.n.10"},{"index":4,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":5,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":6,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":7,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":8,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":9,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":10,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d000.s027.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":810,"cached_input_tokens":0,"output_tokens":303,"reasoning_output_tokens":287},"cost":{"currency":"USD","total_usd":0.013139999999999999,"input_uncached_usd":0.004050000000000001,"input_cached_usd":0.0,"output_usd":0.00909,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.502358625060879}
{"item_id":"semeval2015.d000.s027.t008","gold_sense_keys":["involve%2:42:04::"],"candidates":[{"index":1,"sense_key":"involve%2:42:04::","synset_id":"involve.v.02"},{"index":2,"sense_key":"involve%2:42:01::","synset_id":"involve.v.01"},{"index":3,"sense_key":"involve%2:42:00::","synset_id":"imply.v.05"},{"index":4,"sense_key":"involve%2:42:07::","synset_id":"necessitate.v.01"},{"index":5,"sense_key":"involve%2:42:02::","synset_id":"involve.v.05"},{"index":6,"sense_key":"involve%2:30:00::","synset_id":"involve.v.07"},{"index":7,"sense_key":"involve%2:31:00::","synset_id":"involve.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involve%2:42:04::","call_ids":["semeval2015.d000.s027.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involve%2:42:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":349,"reasoning_output_tokens":333},"cost":{"currency":"USD","total_usd":0.013785,"input_uncached_usd":0.0033150000000000002,"input_cached_usd":0.0,"output_usd":0.01047,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.827787499991246}
{"item_id":"semeval2015.d000.s027.t009","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s027.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.003975,"input_uncached_usd":0.002415,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4611956670414656}
{"item_id":"semeval2015.d000.s027.t010","gold_sense_keys":["locally%4:02:01::"],"candidates":[{"index":1,"sense_key":"locally%4:02:01::","synset_id":"locally.r.02"},{"index":2,"sense_key":"locally%4:02:00::","synset_id":"locally.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"locally%4:02:01::","call_ids":["semeval2015.d000.s027.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"locally%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":136},"cost":{"currency":"USD","total_usd":0.006880000000000001,"input_uncached_usd":0.00232,"input_cached_usd":0.0,"output_usd":0.00456,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.2911170000443235}
{"item_id":"semeval2015.d000.s027.t011","gold_sense_keys":["advanced%5:00:00:late:02"],"candidates":[{"index":1,"sense_key":"advanced%5:00:00:progressive:01","synset_id":"advanced.s.03"},{"index":2,"sense_key":"advanced%5:00:00:civilized:00","synset_id":"advanced.s.07"},{"index":3,"sense_key":"advanced%5:00:00:front:00","synset_id":"advance.s.02"},{"index":4,"sense_key":"advanced%5:00:00:high-tech:00","synset_id":"advanced.s.05"},{"index":5,"sense_key":"advanced%5:00:00:late:02","synset_id":"advanced.s.02"},{"index":6,"sense_key":"advanced%5:00:00:precocious:00","synset_id":"advanced.s.01"},{"index":7,"sense_key":"advanced%5:00:00:late:00","synset_id":"advanced.s.06"},{"index":8,"sense_key":"advanced%5:00:02:high:02","synset_id":"advanced.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"advanced%5:00:00:late:02","call_ids":["semeval2015.d000.s027.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"advanced%5:00:00:late:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":764,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.006070000000000001,"input_uncached_usd":0.0038200000000000005,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8185187089256942}
{"item_id":"semeval2015.d000.s027.t016","gold_sense_keys":["past%1:28:00::"],"candidates":[{"index":1,"sense_key":"past%1:28:01::","synset_id":"past.n.02"},{"index":2,"sense_key":"past%1:10:00::","synset_id":"past.n.03"},{"index":3,"sense_key":"past%1:28:00::","synset_id":"past.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"past%1:28:00::","call_ids":["semeval2015.d000.s027.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"past%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":99},"cost":{"currency":"USD","total_usd":0.005965,"input_uncached_usd":0.0025450000000000004,"input_cached_usd":0.0,"output_usd":0.0034200000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9178228329401463}
{"item_id":"semeval2015.d000.s028.t001","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":2,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"},{"index":3,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":4,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"compare%2:31:00::","call_ids":["semeval2015.d000.s028.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":541,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.005105,"input_uncached_usd":0.0027050000000000004,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.982336250017397}
{"item_id":"semeval2015.d000.s028.t003","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":2,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":3,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s028.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":40,"reasoning_output_tokens":25},"cost":{"currency":"USD","total_usd":0.00392,"input_uncached_usd":0.00272,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6018152500037104}
{"item_id":"semeval2015.d000.s028.t005","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:01::","synset_id":"study.n.10"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":6,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":7,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":8,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":9,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":10,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d000.s028.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":785,"cached_input_tokens":0,"output_tokens":315,"reasoning_output_tokens":299},"cost":{"currency":"USD","total_usd":0.013375000000000001,"input_uncached_usd":0.0039250000000000005,"input_cached_usd":0.0,"output_usd":0.00945,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.457486458006315}
{"item_id":"semeval2015.d000.s028.t006","gold_sense_keys":["involve%2:42:04::"],"candidates":[{"index":1,"sense_key":"involve%2:42:02::","synset_id":"involve.v.05"},{"index":2,"sense_key":"involve%2:31:00::","synset_id":"involve.v.06"},{"index":3,"sense_key":"involve%2:42:07::","synset_id":"necessitate.v.01"},{"index":4,"sense_key":"involve%2:42:04::","synset_id":"involve.v.02"},{"index":5,"sense_key":"involve%2:30:00::","synset_id":"involve.v.07"},{"index":6,"sense_key":"involve%2:42:01::","synset_id":"involve.v.01"},{"index":7,"sense_key":"involve%2:42:00::","synset_id":"imply.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"involve%2:42:04::","call_ids":["semeval2015.d000.s028.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"involve%2:42:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":160,"reasoning_output_tokens":144},"cost":{"currency":"USD","total_usd":0.00799,"input_uncached_usd":0.00319,"input_cached_usd":0.0,"output_usd":0.0048000000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2685844170628116}
{"item_id":"semeval2015.d000.s028.t007","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s028.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":458,"cached_input_tokens":0,"output_tokens":32,"reasoning_output_tokens":17},"cost":{"currency":"USD","total_usd":0.0032500000000000003,"input_uncached_usd":0.0022900000000000004,"input_cached_usd":0.0,"output_usd":0.00096,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.1564845419488847}
{"item_id":"semeval2015.d000.s028.t012","gold_sense_keys":["past%1:28:00::"],"candidates":[{"index":1,"sense_key":"past%1:10:00::","synset_id":"past.n.03"},{"index":2,"sense_key":"past%1:28:00::","synset_id":"past.n.01"},{"index":3,"sense_key":"past%1:28:01::","synset_id":"past.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"past%1:28:00::","call_ids":["semeval2015.d000.s028.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"past%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.00494,"input_uncached_usd":0.0024200000000000003,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3753030829830095}
{"item_id":"semeval2015.d000.s029.t000","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":2,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":3,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":4,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":5,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":6,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":7,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":8,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":9,"sense_key":"study%1:10:01::","synset_id":"study.n.10"},{"index":10,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d000.s029.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":245,"reasoning_output_tokens":229},"cost":{"currency":"USD","total_usd":0.011115,"input_uncached_usd":0.003765,"input_cached_usd":0.0,"output_usd":0.00735,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.245257958071306}
{"item_id":"semeval2015.d000.s029.t001","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":2,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"},{"index":3,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["semeval2015.d000.s029.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.003875,"input_uncached_usd":0.002405,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.310148000018671}
{"item_id":"semeval2015.d000.s029.t002","gold_sense_keys":["measure%1:10:03::"],"candidates":[{"index":1,"sense_key":"measure%1:06:00::","synset_id":"measuring_stick.n.01"},{"index":2,"sense_key":"measure%1:03:00::","synset_id":"measure.n.02"},{"index":3,"sense_key":"measure%1:10:01::","synset_id":"meter.n.03"},{"index":4,"sense_key":"measure%1:04:00::","synset_id":"measurement.n.01"},{"index":5,"sense_key":"measure%1:10:00::","synset_id":"measure.n.07"},{"index":6,"sense_key":"measure%1:04:01::","synset_id":"measure.n.01"},{"index":7,"sense_key":"measure%1:10:02::","synset_id":"bill.n.01"},{"index":8,"sense_key":"measure%1:10:03::","synset_id":"standard.n.01"},{"index":9,"sense_key":"measure%1:06:01::","synset_id":"measure.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"measure%1:10:03::","call_ids":["semeval2015.d000.s029.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"measure%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":211,"reasoning_output_tokens":195},"cost":{"currency":"USD","total_usd":0.010020000000000001,"input_uncached_usd":0.00369,"input_cached_usd":0.0,"output_usd":0.0063300000000000006,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.550822083023377}
{"item_id":"semeval2015.d000.s029.t004","gold_sense_keys":["long%4:02:00::"],"candidates":[{"index":1,"sense_key":"long%4:02:03::","synset_id":"long.r.02"},{"index":2,"sense_key":"long%4:02:00::","synset_id":"long.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"long%4:02:00::","call_ids":["semeval2015.d000.s029.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"long%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":397,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.003935,"input_uncached_usd":0.0019850000000000002,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.599568207981065}
{"item_id":"semeval2015.d000.s029.t005","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s029.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":41,"reasoning_output_tokens":26},"cost":{"currency":"USD","total_usd":0.00336,"input_uncached_usd":0.0021300000000000004,"input_cached_usd":0.0,"output_usd":0.00123,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.376654000021517}
{"item_id":"semeval2015.d000.s029.t006","gold_sense_keys":["survive%2:42:01::"],"candidates":[{"index":1,"sense_key":"survive%2:42:03::","synset_id":"outlive.v.01"},{"index":2,"sense_key":"survive%2:42:01::","synset_id":"survive.v.01"},{"index":3,"sense_key":"survive%2:42:00::","synset_id":"exist.v.02"},{"index":4,"sense_key":"survive%2:42:02::","synset_id":"survive.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"survive%2:42:01::","call_ids":["semeval2015.d000.s029.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"survive%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":554,"cached_input_tokens":0,"output_tokens":434,"reasoning_output_tokens":418},"cost":{"currency":"USD","total_usd":0.015790000000000002,"input_uncached_usd":0.0027700000000000003,"input_cached_usd":0.0,"output_usd":0.01302,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.7348887919215485}
{"item_id":"semeval2015.d000.s030.t000","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["semeval2015.d000.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":436,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.00461,"input_uncached_usd":0.00218,"input_cached_usd":0.0,"output_usd":0.00243,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.612573624937795}
{"item_id":"semeval2015.d000.s030.t001","gold_sense_keys":["show%2:32:00::","show%2:39:00::"],"candidates":[{"index":1,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":2,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":5,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":6,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":7,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":8,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":9,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":10,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":11,"sense_key":"show%2:33:00::","synset_id":"show.v.12"},{"index":12,"sense_key":"show%2:39:02::","synset_id":"show.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"show%2:31:00::","call_ids":["semeval2015.d000.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"show%2:31:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":918,"cached_input_tokens":0,"output_tokens":1564,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.05151,"input_uncached_usd":0.00459,"input_cached_usd":0.0,"output_usd":0.04692,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":30.10508645803202}
{"item_id":"semeval2015.d000.s030.t002","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":2,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":3,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:10:01::","synset_id":"study.n.10"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":8,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:06:00::","synset_id":"study.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d000.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":280,"reasoning_output_tokens":265},"cost":{"currency":"USD","total_usd":0.012210000000000002,"input_uncached_usd":0.0038100000000000005,"input_cached_usd":0.0,"output_usd":0.0084,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.403147874982096}
{"item_id":"semeval2015.d000.s031.t000","gold_sense_keys":["increase%2:30:02::"],"candidates":[{"index":1,"sense_key":"increase%2:30:02::","synset_id":"increase.v.02"},{"index":2,"sense_key":"increase%2:30:00::","synset_id":"increase.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"increase%2:30:02::","call_ids":["semeval2015.d000.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"increase%2:30:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":421,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.003905,"input_uncached_usd":0.002105,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5638687080936506}
{"item_id":"semeval2015.d000.s031.t001","gold_sense_keys":["survival%1:26:00::"],"candidates":[{"index":1,"sense_key":"survival%1:26:00::","synset_id":"survival.n.01"},{"index":2,"sense_key":"survival%1:22:00::","synset_id":"survival.n.02"},{"index":3,"sense_key":"survival%1:04:00::","synset_id":"survival.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survival%1:26:00::","call_ids":["semeval2015.d000.s031.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survival%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0038950000000000005,"input_uncached_usd":0.0022150000000000004,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7953526659402996}
{"item_id":"semeval2015.d000.s031.t002","gold_sense_keys":["time%1:28:00::"],"candidates":[{"index":1,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":2,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":3,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":4,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":5,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":8,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"},{"index":9,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":10,"sense_key":"time%1:03:00::","synset_id":"time.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"time%1:28:00::","call_ids":["semeval2015.d000.s031.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"time%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":140},"cost":{"currency":"USD","total_usd":0.008445000000000001,"input_uncached_usd":0.003765,"input_cached_usd":0.0,"output_usd":0.00468,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.253767332993448}
{"item_id":"semeval2015.d000.s031.t003","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s031.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.0042450000000000005,"input_uncached_usd":0.0021750000000000003,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.477397041977383}
{"item_id":"semeval2015.d000.s032.t000","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"},{"index":2,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s032.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":436,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.00371,"input_uncached_usd":0.00218,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9234576250892133}
{"item_id":"semeval2015.d000.s032.t002","gold_sense_keys":["survive%2:42:01::"],"candidates":[{"index":1,"sense_key":"survive%2:42:00::","synset_id":"exist.v.02"},{"index":2,"sense_key":"survive%2:42:01::","synset_id":"survive.v.01"},{"index":3,"sense_key":"survive%2:42:02::","synset_id":"survive.v.02"},{"index":4,"sense_key":"survive%2:42:03::","synset_id":"outlive.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"survive%2:42:01::","call_ids":["semeval2015.d000.s032.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"survive%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018665,"input_uncached_usd":0.0028250000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.041399957961403}
{"item_id":"semeval2015.d000.s032.t003","gold_sense_keys":["average%1:09:00::"],"candidates":[{"index":1,"sense_key":"average%1:09:00::","synset_id":"average.n.01"},{"index":2,"sense_key":"average%1:24:00::","synset_id":"average.n.02"},{"index":3,"sense_key":"average%1:09:01::","synset_id":"average.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"average%1:09:00::","call_ids":["semeval2015.d000.s032.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"average%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":114},"cost":{"currency":"USD","total_usd":0.006180000000000001,"input_uncached_usd":0.0022800000000000003,"input_cached_usd":0.0,"output_usd":0.0039000000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.779845208977349}
{"item_id":"semeval2015.d000.s032.t004","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:00::","synset_id":"month.n.02"},{"index":2,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"month%1:28:00::","call_ids":["semeval2015.d000.s032.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.00422,"input_uncached_usd":0.0021200000000000004,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0697308339877054}
{"item_id":"semeval2015.d000.s032.t005","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":2,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":3,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"},{"index":4,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compare%2:31:00::","call_ids":["semeval2015.d000.s032.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":248,"reasoning_output_tokens":232},"cost":{"currency":"USD","total_usd":0.010035,"input_uncached_usd":0.002595,"input_cached_usd":0.0,"output_usd":0.00744,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.439360499964096}
{"item_id":"semeval2015.d000.s032.t006","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:00::","synset_id":"month.n.02"},{"index":2,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"month%1:28:00::","call_ids":["semeval2015.d000.s032.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":425,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004135,"input_uncached_usd":0.002125,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1570230410434306}
{"item_id":"semeval2015.d000.s033.t000","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d000.s033.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004540000000000001,"input_uncached_usd":0.00256,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5990331249777228}
{"item_id":"semeval2015.d000.s033.t001","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"},{"index":2,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s033.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":32,"reasoning_output_tokens":17},"cost":{"currency":"USD","total_usd":0.0030700000000000002,"input_uncached_usd":0.0021100000000000003,"input_cached_usd":0.0,"output_usd":0.00096,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.147154416074045}
{"item_id":"semeval2015.d000.s033.t002","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"},{"index":2,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":3,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":4,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":5,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"cell%1:03:00::","call_ids":["semeval2015.d000.s033.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":608,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.0052,"input_uncached_usd":0.00304,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7677858331007883}
{"item_id":"semeval2015.d000.s033.t005","gold_sense_keys":["effective%3:00:00::","effective%5:00:00:competent:00","effective%5:00:00:efficacious:00"],"candidates":[{"index":1,"sense_key":"effective%5:00:00:competent:00","synset_id":"effective.s.01"},{"index":2,"sense_key":"effective%5:00:00:efficacious:00","synset_id":"effective.s.02"},{"index":3,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":4,"sense_key":"effective%5:00:00:actual:00","synset_id":"effective.s.04"},{"index":5,"sense_key":"effective%3:00:00::","synset_id":"effective.a.01"},{"index":6,"sense_key":"effective%5:00:00:operational:00","synset_id":"effective.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"effective%5:00:00:efficacious:00","call_ids":["semeval2015.d000.s033.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"effective%5:00:00:efficacious:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":656,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01912,"input_uncached_usd":0.0032800000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":17.666513124946505}
{"item_id":"semeval2015.d000.s034.t000","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s034.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":401,"cached_input_tokens":0,"output_tokens":43,"reasoning_output_tokens":28},"cost":{"currency":"USD","total_usd":0.003295,"input_uncached_usd":0.0020050000000000003,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5658361670793965}
{"item_id":"semeval2015.d000.s034.t003","gold_sense_keys":["past%1:28:00::"],"candidates":[{"index":1,"sense_key":"past%1:10:00::","synset_id":"past.n.03"},{"index":2,"sense_key":"past%1:28:01::","synset_id":"past.n.02"},{"index":3,"sense_key":"past%1:28:00::","synset_id":"past.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"past%1:28:00::","call_ids":["semeval2015.d000.s034.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"past%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.003965,"input_uncached_usd":0.002135,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9752431659726426}
{"item_id":"semeval2015.d000.s034.t004","gold_sense_keys":["average%5:00:01:normal:01","average%5:00:02:normal:01","average%5:00:03:normal:01"],"candidates":[{"index":1,"sense_key":"average%5:00:00:common:01","synset_id":"average.s.02"},{"index":2,"sense_key":"average%5:00:00:ordinary:00","synset_id":"average.s.03"},{"index":3,"sense_key":"average%5:00:01:normal:01","synset_id":"median.s.01"},{"index":4,"sense_key":"average%5:00:03:normal:01","synset_id":"modal.s.01"},{"index":5,"sense_key":"average%5:00:02:normal:01","synset_id":"average.s.01"},{"index":6,"sense_key":"average%5:00:00:moderate:00","synset_id":"average.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"average%5:00:02:normal:01","call_ids":["semeval2015.d000.s034.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"average%5:00:02:normal:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019145,"input_uncached_usd":0.003305,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.626084625022486}
{"item_id":"semeval2015.d000.s034.t005","gold_sense_keys":["survival%1:26:00::"],"candidates":[{"index":1,"sense_key":"survival%1:04:00::","synset_id":"survival.n.03"},{"index":2,"sense_key":"survival%1:26:00::","synset_id":"survival.n.01"},{"index":3,"sense_key":"survival%1:22:00::","synset_id":"survival.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"survival%1:26:00::","call_ids":["semeval2015.d000.s034.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"survival%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.003785,"input_uncached_usd":0.0020450000000000004,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7101889590267092}
{"item_id":"semeval2015.d000.s034.t006","gold_sense_keys":["time%1:28:00::"],"candidates":[{"index":1,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":2,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":3,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":4,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":5,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":6,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":7,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":8,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"time%1:28:00::","call_ids":["semeval2015.d000.s034.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"time%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":719,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":134},"cost":{"currency":"USD","total_usd":0.008095000000000001,"input_uncached_usd":0.003595,"input_cached_usd":0.0,"output_usd":0.0045000000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.82507491693832}
{"item_id":"semeval2015.d000.s034.t007","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:00::","synset_id":"month.n.02"},{"index":2,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"month%1:28:00::","call_ids":["semeval2015.d000.s034.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.0035399999999999997,"input_uncached_usd":0.0019500000000000001,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6819270409177989}
{"item_id":"semeval2015.d000.s034.t008","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":2,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":3,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"},{"index":4,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"compare%2:31:00::","call_ids":["semeval2015.d000.s034.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":130},"cost":{"currency":"USD","total_usd":0.0068000000000000005,"input_uncached_usd":0.0024200000000000003,"input_cached_usd":0.0,"output_usd":0.00438,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2378590831067413}
{"item_id":"semeval2015.d000.s034.t009","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["semeval2015.d000.s034.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.00408,"input_uncached_usd":0.0019500000000000001,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7752321669831872}
{"item_id":"semeval2015.d000.s035.t000","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"},{"index":2,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s035.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.00367,"input_uncached_usd":0.0020800000000000003,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6450349999358878}
{"item_id":"semeval2015.d000.s035.t004","gold_sense_keys":["past%1:28:00::"],"candidates":[{"index":1,"sense_key":"past%1:28:01::","synset_id":"past.n.02"},{"index":2,"sense_key":"past%1:10:00::","synset_id":"past.n.03"},{"index":3,"sense_key":"past%1:28:00::","synset_id":"past.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"past%1:28:00::","call_ids":["semeval2015.d000.s035.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"past%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":93},"cost":{"currency":"USD","total_usd":0.0054800000000000005,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.00327,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5794756249524653}
{"item_id":"semeval2015.d000.s035.t005","gold_sense_keys":["average%5:00:01:normal:01","average%5:00:02:normal:01","average%5:00:03:normal:01"],"candidates":[{"index":1,"sense_key":"average%5:00:00:ordinary:00","synset_id":"average.s.03"},{"index":2,"sense_key":"average%5:00:01:normal:01","synset_id":"median.s.01"},{"index":3,"sense_key":"average%5:00:03:normal:01","synset_id":"modal.s.01"},{"index":4,"sense_key":"average%5:00:00:moderate:00","synset_id":"average.s.04"},{"index":5,"sense_key":"average%5:00:02:normal:01","synset_id":"average.s.01"},{"index":6,"sense_key":"average%5:00:00:common:01","synset_id":"average.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"average%5:00:02:normal:01","call_ids":["semeval2015.d000.s035.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"average%5:00:02:normal:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":676,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01922,"input_uncached_usd":0.00338,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.079229083028622}
{"item_id":"semeval2015.d000.s035.t006","gold_sense_keys":["survival%1:26:00::"],"candidates":[{"index":1,"sense_key":"survival%1:26:00::","synset_id":"survival.n.01"},{"index":2,"sense_key":"survival%1:22:00::","synset_id":"survival.n.02"},{"index":3,"sense_key":"survival%1:04:00::","synset_id":"survival.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survival%1:26:00::","call_ids":["semeval2015.d000.s035.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survival%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.0037700000000000003,"input_uncached_usd":0.0021200000000000004,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4501017919974402}
{"item_id":"semeval2015.d000.s035.t007","gold_sense_keys":["time%1:28:00::"],"candidates":[{"index":1,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":2,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":3,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":4,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":5,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":6,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":7,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":8,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"},{"index":9,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":10,"sense_key":"time%1:28:05::","synset_id":"time.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"time%1:28:00::","call_ids":["semeval2015.d000.s035.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"time%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":734,"cached_input_tokens":0,"output_tokens":158,"reasoning_output_tokens":143},"cost":{"currency":"USD","total_usd":0.00841,"input_uncached_usd":0.00367,"input_cached_usd":0.0,"output_usd":0.00474,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8889699999708682}
{"item_id":"semeval2015.d000.s035.t008","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["semeval2015.d000.s035.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":405,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.004335,"input_uncached_usd":0.0020250000000000003,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6997084999457002}
{"item_id":"semeval2015.d000.s035.t009","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":2,"sense_key":"group%1:09:00::","synset_id":"group.n.03"},{"index":3,"sense_key":"group%1:03:00::","synset_id":"group.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"group%1:03:00::","call_ids":["semeval2015.d000.s035.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.004450000000000001,"input_uncached_usd":0.00223,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0573641669470817}
{"item_id":"semeval2015.d000.s036.t000","gold_sense_keys":["however%4:02:04::"],"candidates":[{"index":1,"sense_key":"however%4:02:04::","synset_id":"however.r.02"},{"index":2,"sense_key":"however%4:02:01::","synset_id":"however.r.04"},{"index":3,"sense_key":"however%4:02:00::","synset_id":"however.r.01"},{"index":4,"sense_key":"however%4:02:02::","synset_id":"however.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"however%4:02:04::","call_ids":["semeval2015.d000.s036.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"however%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01858,"input_uncached_usd":0.0027400000000000002,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.591312582953833}
{"item_id":"semeval2015.d000.s036.t001","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":2,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":3,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":4,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":5,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":6,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":7,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":8,"sense_key":"study%1:10:01::","synset_id":"study.n.10"},{"index":9,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":10,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d000.s036.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":205,"reasoning_output_tokens":189},"cost":{"currency":"USD","total_usd":0.009920000000000002,"input_uncached_usd":0.0037700000000000003,"input_cached_usd":0.0,"output_usd":0.00615,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.689207791001536}
{"item_id":"semeval2015.d000.s036.t002","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s036.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":40,"reasoning_output_tokens":25},"cost":{"currency":"USD","total_usd":0.0033350000000000003,"input_uncached_usd":0.002135,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4407290410017595}
{"item_id":"semeval2015.d000.s036.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":2,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":5,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d000.s036.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":541,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.004895,"input_uncached_usd":0.0027050000000000004,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7509628749685362}
{"item_id":"semeval2015.d000.s036.t005","gold_sense_keys":["affect%2:29:00::"],"candidates":[{"index":1,"sense_key":"affect%2:37:00::","synset_id":"affect.v.05"},{"index":2,"sense_key":"affect%2:29:00::","synset_id":"affect.v.02"},{"index":3,"sense_key":"affect%2:32:00::","synset_id":"feign.v.01"},{"index":4,"sense_key":"affect%2:30:00::","synset_id":"affect.v.01"},{"index":5,"sense_key":"affect%2:42:00::","synset_id":"involve.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"affect%2:29:00::","call_ids":["semeval2015.d000.s036.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"affect%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.03422,"input_uncached_usd":0.00284,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":15.226634667022154}
{"item_id":"semeval2015.d000.s036.t008","gold_sense_keys":["long%3:00:02::"],"candidates":[{"index":1,"sense_key":"long%5:00:00:tall:00","synset_id":"long.s.01"},{"index":2,"sense_key":"long%3:00:05::","synset_id":"long.a.05"},{"index":3,"sense_key":"long%3:00:01::","synset_id":"long.a.02"},{"index":4,"sense_key":"long%3:00:02::","synset_id":"long.a.01"},{"index":5,"sense_key":"long%3:00:04::","synset_id":"long.a.06"},{"index":6,"sense_key":"long%3:00:00::","synset_id":"retentive.a.01"},{"index":7,"sense_key":"long%5:00:00:unsound:00","synset_id":"long.s.02"},{"index":8,"sense_key":"long%5:00:00:provident:00","synset_id":"farseeing.s.02"},{"index":9,"sense_key":"long%5:00:00:abundant:00","synset_id":"long.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"long%3:00:02::","call_ids":["semeval2015.d000.s036.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"long%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":755,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.005305,"input_uncached_usd":0.003775,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5315579579910263}
{"item_id":"semeval2015.d000.s036.t009","gold_sense_keys":["survival%1:26:00::"],"candidates":[{"index":1,"sense_key":"survival%1:26:00::","synset_id":"survival.n.01"},{"index":2,"sense_key":"survival%1:22:00::","synset_id":"survival.n.02"},{"index":3,"sense_key":"survival%1:04:00::","synset_id":"survival.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survival%1:26:00::","call_ids":["semeval2015.d000.s036.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survival%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":32},"cost":{"currency":"USD","total_usd":0.0035850000000000005,"input_uncached_usd":0.0021750000000000003,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3522893339395523}
{"item_id":"semeval2015.d000.s036.t010","gold_sense_keys":["time%1:28:00::"],"candidates":[{"index":1,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":2,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":3,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":4,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":5,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":6,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":7,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"},{"index":8,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":9,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":10,"sense_key":"time%1:11:00::","synset_id":"time.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"time%1:28:00::","call_ids":["semeval2015.d000.s036.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"time%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":745,"cached_input_tokens":0,"output_tokens":165,"reasoning_output_tokens":150},"cost":{"currency":"USD","total_usd":0.008675,"input_uncached_usd":0.0037250000000000004,"input_cached_usd":0.0,"output_usd":0.00495,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.210010625072755}
{"item_id":"semeval2015.d000.s037.t000","gold_sense_keys":["contrast%1:24:00::"],"candidates":[{"index":1,"sense_key":"contrast%1:09:01::","synset_id":"line.n.29"},{"index":2,"sense_key":"contrast%1:24:00::","synset_id":"contrast.n.01"},{"index":3,"sense_key":"contrast%1:07:00::","synset_id":"contrast.n.05"},{"index":4,"sense_key":"contrast%1:04:00::","synset_id":"contrast.n.02"},{"index":5,"sense_key":"contrast%1:09:00::","synset_id":"contrast.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"contrast%1:24:00::","call_ids":["semeval2015.d000.s037.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"contrast%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.00523,"input_uncached_usd":0.00253,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9202240830054507}
{"item_id":"semeval2015.d000.s037.t001","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"},{"index":2,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s037.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.0036100000000000004,"input_uncached_usd":0.00205,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6030551670119166}
{"item_id":"semeval2015.d000.s037.t002","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":2,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":3,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":4,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":5,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d000.s037.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":37,"reasoning_output_tokens":22},"cost":{"currency":"USD","total_usd":0.0037300000000000002,"input_uncached_usd":0.0026200000000000004,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.1006064580287784}
{"item_id":"semeval2015.d000.s037.t003","gold_sense_keys":["affect%2:29:00::"],"candidates":[{"index":1,"sense_key":"affect%2:37:00::","synset_id":"affect.v.05"},{"index":2,"sense_key":"affect%2:30:00::","synset_id":"affect.v.01"},{"index":3,"sense_key":"affect%2:42:00::","synset_id":"involve.v.01"},{"index":4,"sense_key":"affect%2:32:00::","synset_id":"feign.v.01"},{"index":5,"sense_key":"affect%2:29:00::","synset_id":"affect.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"affect%2:29:00::","call_ids":["semeval2015.d000.s037.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"affect%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018595,"input_uncached_usd":0.002755,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.313412541989237}
{"item_id":"semeval2015.d000.s037.t006","gold_sense_keys":["short%3:00:02::"],"candidates":[{"index":1,"sense_key":"short%5:00:00:improvident:00","synset_id":"short.s.03"},{"index":2,"sense_key":"short%3:00:02::","synset_id":"short.a.01"},{"index":3,"sense_key":"short%3:00:00::","synset_id":"unretentive.a.01"},{"index":4,"sense_key":"short%5:00:02:insufficient:00","synset_id":"light.s.10"},{"index":5,"sense_key":"short%5:00:00:discourteous:00","synset_id":"brusque.s.01"},{"index":6,"sense_key":"short%3:00:01::","synset_id":"short.a.02"},{"index":7,"sense_key":"short%3:00:04::","synset_id":"short.a.07"},{"index":8,"sense_key":"short%3:00:05::","synset_id":"short.a.06"},{"index":9,"sense_key":"short%5:00:00:breakable:00","synset_id":"short.s.04"},{"index":10,"sense_key":"short%3:00:03::","synset_id":"short.a.03"},{"index":11,"sense_key":"short%5:00:00:insufficient:00","synset_id":"inadequate.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"short%3:00:02::","call_ids":["semeval2015.d000.s037.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"short%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.00618,"input_uncached_usd":0.00417,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.727130917017348}
{"item_id":"semeval2015.d000.s037.t007","gold_sense_keys":["survival%1:26:00::"],"candidates":[{"index":1,"sense_key":"survival%1:04:00::","synset_id":"survival.n.03"},{"index":2,"sense_key":"survival%1:26:00::","synset_id":"survival.n.01"},{"index":3,"sense_key":"survival%1:22:00::","synset_id":"survival.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"survival%1:26:00::","call_ids":["semeval2015.d000.s037.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"survival%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.004070000000000001,"input_uncached_usd":0.0020900000000000003,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.51419287500903}
{"item_id":"semeval2015.d000.s037.t008","gold_sense_keys":["time%1:28:00::"],"candidates":[{"index":1,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":2,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":3,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":6,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":7,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"},{"index":8,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":9,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":10,"sense_key":"time%1:28:05::","synset_id":"time.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"time%1:28:00::","call_ids":["semeval2015.d000.s037.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"time%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":121},"cost":{"currency":"USD","total_usd":0.00775,"input_uncached_usd":0.0036400000000000004,"input_cached_usd":0.0,"output_usd":0.00411,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1922379590105265}
{"item_id":"semeval2015.d001.s000.t000","gold_sense_keys":["mathematical%3:01:00::"],"candidates":[{"index":1,"sense_key":"mathematical%3:01:00::","synset_id":"mathematical.a.01"},{"index":2,"sense_key":"mathematical%5:00:00:possible:00","synset_id":"mathematical.s.02"},{"index":3,"sense_key":"mathematical%5:00:00:exact:00","synset_id":"mathematical.s.03"},{"index":4,"sense_key":"mathematical%3:00:00::","synset_id":"numerical.a.03"},{"index":5,"sense_key":"mathematical%5:00:00:unquestionable:00","synset_id":"mathematical.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mathematical%3:01:00::","call_ids":["semeval2015.d001.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mathematical%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":393,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0038550000000000004,"input_uncached_usd":0.001965,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5764967920258641}
{"item_id":"semeval2015.d001.s000.t001","gold_sense_keys":["base%2:31:00::"],"candidates":[{"index":1,"sense_key":"base%2:31:00::","synset_id":"establish.v.08"},{"index":2,"sense_key":"base%2:42:04::","synset_id":"base.v.02"},{"index":3,"sense_key":"base%2:34:00::","synset_id":"free-base.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"base%2:31:00::","call_ids":["semeval2015.d001.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"base%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":306,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.00321,"input_uncached_usd":0.0015300000000000001,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5888240840286016}
{"item_id":"semeval2015.d001.s001.t001","gold_sense_keys":["capable%3:00:00::"],"candidates":[{"index":1,"sense_key":"capable%3:00:00::","synset_id":"capable.a.01"},{"index":2,"sense_key":"capable%5:00:00:competent:00","synset_id":"able.s.01"},{"index":3,"sense_key":"capable%5:00:00:susceptible:00","synset_id":"capable.s.01"},{"index":4,"sense_key":"capable%3:00:02::","synset_id":"capable.a.03"},{"index":5,"sense_key":"capable%5:00:00:adequate:00","synset_id":"adequate_to.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"capable%3:00:00::","call_ids":["semeval2015.d001.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"capable%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":430,"cached_input_tokens":0,"output_tokens":205,"reasoning_output_tokens":189},"cost":{"currency":"USD","total_usd":0.008300000000000002,"input_uncached_usd":0.00215,"input_cached_usd":0.0,"output_usd":0.00615,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.2089140000753105}
{"item_id":"semeval2015.d001.s001.t002","gold_sense_keys":["make%2:36:12::"],"candidates":[{"index":1,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":2,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":3,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":4,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":5,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":6,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":7,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":8,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":9,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":10,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":11,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":12,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":13,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":14,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":15,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":16,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":17,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":18,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":19,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":20,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":21,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":22,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":23,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":24,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":25,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":26,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":27,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":30,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":31,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":32,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":33,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":34,"sense_key":"make%2:29:01::","synset_id":"make.v.49"},{"index":35,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":36,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":37,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":38,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":39,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":40,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":41,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":42,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":43,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":44,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":45,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":46,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":47,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":48,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":49,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"make%2:36:12::","call_ids":["semeval2015.d001.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"make%2:36:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2162,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":130},"cost":{"currency":"USD","total_usd":0.01519,"input_uncached_usd":0.01081,"input_cached_usd":0.0,"output_usd":0.00438,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8755101669812575}
{"item_id":"semeval2015.d001.s001.t003","gold_sense_keys":["simple%3:00:02::","simple%5:00:00:easy:01"],"candidates":[{"index":1,"sense_key":"simple%5:00:00:plain:01","synset_id":"simple.s.05"},{"index":2,"sense_key":"simple%3:00:02::","synset_id":"simple.a.01"},{"index":3,"sense_key":"simple%5:00:00:easy:01","synset_id":"elementary.s.01"},{"index":4,"sense_key":"simple%5:00:02:plain:01","synset_id":"bare.s.04"},{"index":5,"sense_key":"simple%3:00:01::","synset_id":"simple.a.06"},{"index":6,"sense_key":"simple%5:00:00:naive:00","synset_id":"childlike.s.02"},{"index":7,"sense_key":"simple%5:00:00:retarded:00","synset_id":"dim-witted.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"simple%3:00:02::","call_ids":["semeval2015.d001.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"simple%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":498,"reasoning_output_tokens":482},"cost":{"currency":"USD","total_usd":0.01749,"input_uncached_usd":0.00255,"input_cached_usd":0.0,"output_usd":0.01494,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.893562417011708}
{"item_id":"semeval2015.d001.s001.t004","gold_sense_keys":["operation%1:04:01::"],"candidates":[{"index":1,"sense_key":"operation%1:22:01::","synset_id":"operation.n.08"},{"index":2,"sense_key":"operation%1:22:00::","synset_id":"operation.n.04"},{"index":3,"sense_key":"operation%1:04:02::","synset_id":"operation.n.05"},{"index":4,"sense_key":"operation%1:04:01::","synset_id":"mathematical_process.n.01"},{"index":5,"sense_key":"operation%1:04:07::","synset_id":"operation.n.11"},{"index":6,"sense_key":"operation%1:04:03::","synset_id":"operation.n.03"},{"index":7,"sense_key":"operation%1:04:00::","synset_id":"operation.n.06"},{"index":8,"sense_key":"operation%1:04:06::","synset_id":"operation.n.02"},{"index":9,"sense_key":"operation%1:26:00::","synset_id":"operation.n.01"},{"index":10,"sense_key":"operation%1:04:05::","synset_id":"operation.n.07"},{"index":11,"sense_key":"operation%1:09:00::","synset_id":"process.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"operation%1:04:01::","call_ids":["semeval2015.d001.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"operation%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":126},"cost":{"currency":"USD","total_usd":0.008085,"input_uncached_usd":0.0038250000000000003,"input_cached_usd":0.0,"output_usd":0.00426,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.49788629100658}
{"item_id":"semeval2015.d001.s001.t006","gold_sense_keys":["3d%1:09:00::"],"candidates":[{"index":1,"sense_key":"3-d%1:09:00::","synset_id":"three-d.n.02"},{"index":2,"sense_key":"3-d%1:10:00::","synset_id":"three-d.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"3-d%1:09:00::","call_ids":["semeval2015.d001.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"3-d%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":281,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.004195,"input_uncached_usd":0.0014050000000000002,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0518355829408392}
{"item_id":"semeval2015.d001.s002.t000","gold_sense_keys":["actually%4:02:00::","actually%4:02:02::"],"candidates":[{"index":1,"sense_key":"actually%4:02:03::","synset_id":"actually.r.03"},{"index":2,"sense_key":"actually%4:02:00::","synset_id":"actually.r.02"},{"index":3,"sense_key":"actually%4:02:02::","synset_id":"actually.r.04"},{"index":4,"sense_key":"actually%4:02:01::","synset_id":"actually.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"actually%4:02:00::","call_ids":["semeval2015.d001.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"actually%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01782,"input_uncached_usd":0.00198,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.03190750000067}
{"item_id":"semeval2015.d001.s002.t002","gold_sense_keys":["necessary%3:00:00::"],"candidates":[{"index":1,"sense_key":"necessary%5:00:00:inevitable:00","synset_id":"necessary.s.01"},{"index":2,"sense_key":"necessary%3:00:00::","synset_id":"necessary.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"necessary%3:00:00::","call_ids":["semeval2015.d001.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"necessary%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":273,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.003345,"input_uncached_usd":0.0013650000000000001,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.976784082944505}
{"item_id":"semeval2015.d001.s002.t003","gold_sense_keys":["know%2:31:15::"],"candidates":[{"index":1,"sense_key":"know%2:31:14::","synset_id":"know.v.11"},{"index":2,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":3,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":4,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":5,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":6,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":7,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":8,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":9,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":10,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":11,"sense_key":"know%2:31:03::","synset_id":"know.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"know%2:31:15::","call_ids":["semeval2015.d001.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"know%2:31:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":100},"cost":{"currency":"USD","total_usd":0.007130000000000001,"input_uncached_usd":0.0036500000000000005,"input_cached_usd":0.0,"output_usd":0.00348,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9364056669874117}
{"item_id":"semeval2015.d001.s002.t004","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":2,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":3,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":6,"sense_key":"use%2:34:01::","synset_id":"use.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.004435,"input_uncached_usd":0.0024850000000000002,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.568964667036198}
{"item_id":"semeval2015.d001.s004.t002","gold_sense_keys":["now%4:02:00::","now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":2,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":3,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":4,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":5,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":6,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":7,"sense_key":"now%4:02:06::","synset_id":"now.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"now%4:02:04::","call_ids":["semeval2015.d001.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"now%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0186,"input_uncached_usd":0.0027600000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.775961208972149}
{"item_id":"semeval2015.d001.s004.t003","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":2,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":6,"sense_key":"use%2:34:01::","synset_id":"use.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.004785,"input_uncached_usd":0.002625,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8686011250829324}
{"item_id":"semeval2015.d001.s004.t004","gold_sense_keys":["little%3:00:03::"],"candidates":[{"index":1,"sense_key":"little%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":2,"sense_key":"little%5:00:00:young:00","synset_id":"little.s.01"},{"index":3,"sense_key":"little%3:00:03::","synset_id":"little.a.02"},{"index":4,"sense_key":"little%5:00:00:unimportant:00","synset_id":"fiddling.s.01"},{"index":5,"sense_key":"little%3:00:00::","synset_id":"short.a.03"},{"index":6,"sense_key":"little%5:00:00:emotional:00","synset_id":"little.s.05"},{"index":7,"sense_key":"little%5:00:00:soft:04","synset_id":"little.s.03"},{"index":8,"sense_key":"little%3:00:01::","synset_id":"small.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"little%3:00:03::","call_ids":["semeval2015.d001.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"little%3:00:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.005540000000000001,"input_uncached_usd":0.0031400000000000004,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8322604580316693}
{"item_id":"semeval2015.d001.s004.t005","gold_sense_keys":["mathematical%3:01:00::"],"candidates":[{"index":1,"sense_key":"mathematical%5:00:00:exact:00","synset_id":"mathematical.s.03"},{"index":2,"sense_key":"mathematical%3:00:00::","synset_id":"numerical.a.03"},{"index":3,"sense_key":"mathematical%3:01:00::","synset_id":"mathematical.a.01"},{"index":4,"sense_key":"mathematical%5:00:00:unquestionable:00","synset_id":"mathematical.s.01"},{"index":5,"sense_key":"mathematical%5:00:00:possible:00","synset_id":"mathematical.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"mathematical%3:01:00::","call_ids":["semeval2015.d001.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"mathematical%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":119},"cost":{"currency":"USD","total_usd":0.0062450000000000006,"input_uncached_usd":0.002225,"input_cached_usd":0.0,"output_usd":0.00402,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.189089125022292}
{"item_id":"semeval2015.d001.s005.t000","gold_sense_keys":["here%4:02:00::"],"candidates":[{"index":1,"sense_key":"here%4:02:04::","synset_id":"here.r.04"},{"index":2,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":3,"sense_key":"here%4:02:00::","synset_id":"here.r.01"},{"index":4,"sense_key":"here%4:02:02::","synset_id":"here.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"here%4:02:00::","call_ids":["semeval2015.d001.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"here%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":122},"cost":{"currency":"USD","total_usd":0.0061200000000000004,"input_uncached_usd":0.00201,"input_cached_usd":0.0,"output_usd":0.00411,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.862259667017497}
{"item_id":"semeval2015.d001.s005.t001","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"},{"index":2,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":3,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["semeval2015.d001.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.003595,"input_uncached_usd":0.0019450000000000001,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.492967874975875}
{"item_id":"semeval2015.d001.s005.t002","gold_sense_keys":["window%1:06:03::"],"candidates":[{"index":1,"sense_key":"window%1:06:03::","synset_id":"window.n.08"},{"index":2,"sense_key":"window%1:28:00::","synset_id":"window.n.05"},{"index":3,"sense_key":"window%1:17:00::","synset_id":"window.n.04"},{"index":4,"sense_key":"window%1:06:01::","synset_id":"window.n.02"},{"index":5,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":6,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":7,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":8,"sense_key":"window%1:06:00::","synset_id":"window.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"window%1:06:03::","call_ids":["semeval2015.d001.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"window%1:06:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":576,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.00447,"input_uncached_usd":0.00288,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.15668958297465}
{"item_id":"semeval2015.d001.s006.t000","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":2,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":3,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["semeval2015.d001.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.00363,"input_uncached_usd":0.00201,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3131748749874532}
{"item_id":"semeval2015.d001.s006.t001","gold_sense_keys":["window%1:06:03::"],"candidates":[{"index":1,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":2,"sense_key":"window%1:06:00::","synset_id":"window.n.01"},{"index":3,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":4,"sense_key":"window%1:06:03::","synset_id":"window.n.08"},{"index":5,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":6,"sense_key":"window%1:06:01::","synset_id":"window.n.02"},{"index":7,"sense_key":"window%1:28:00::","synset_id":"window.n.05"},{"index":8,"sense_key":"window%1:17:00::","synset_id":"window.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"window%1:06:03::","call_ids":["semeval2015.d001.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"window%1:06:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.005165,"input_uncached_usd":0.002945,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.805183917051181}
{"item_id":"semeval2015.d001.s007.t000","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":2,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":3,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["semeval2015.d001.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":391,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.004055,"input_uncached_usd":0.001955,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6760956249199808}
{"item_id":"semeval2015.d001.s007.t001","gold_sense_keys":["window%1:06:03::"],"candidates":[{"index":1,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":2,"sense_key":"window%1:06:03::","synset_id":"window.n.08"},{"index":3,"sense_key":"window%1:06:00::","synset_id":"window.n.01"},{"index":4,"sense_key":"window%1:28:00::","synset_id":"window.n.05"},{"index":5,"sense_key":"window%1:06:01::","synset_id":"window.n.02"},{"index":6,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":7,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":8,"sense_key":"window%1:17:00::","synset_id":"window.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"window%1:06:03::","call_ids":["semeval2015.d001.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"window%1:06:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.00445,"input_uncached_usd":0.00289,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.437455124920234}
{"item_id":"semeval2015.d001.s007.t002","gold_sense_keys":["consist%2:42:01::"],"candidates":[{"index":1,"sense_key":"consist%2:42:02::","synset_id":"consist.v.03"},{"index":2,"sense_key":"consist%2:42:04::","synset_id":"consist.v.02"},{"index":3,"sense_key":"consist%2:42:00::","synset_id":"dwell.v.02"},{"index":4,"sense_key":"consist%2:42:01::","synset_id":"consist.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"consist%2:42:01::","call_ids":["semeval2015.d001.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"consist%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":285,"reasoning_output_tokens":269},"cost":{"currency":"USD","total_usd":0.010715,"input_uncached_usd":0.0021650000000000003,"input_cached_usd":0.0,"output_usd":0.00855,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.734248999971896}
{"item_id":"semeval2015.d001.s007.t003","gold_sense_keys":["console%1:06:02::"],"candidates":[{"index":1,"sense_key":"console%1:06:00::","synset_id":"cabinet.n.04"},{"index":2,"sense_key":"console%1:06:01::","synset_id":"console.n.03"},{"index":3,"sense_key":"console%1:06:02::","synset_id":"console.n.02"},{"index":4,"sense_key":"console%1:06:03::","synset_id":"console_table.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"console%1:06:02::","call_ids":["semeval2015.d001.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"console%1:06:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.00515,"input_uncached_usd":0.002,"input_cached_usd":0.0,"output_usd":0.00315,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3289481250103563}
{"item_id":"semeval2015.d001.s007.t005","gold_sense_keys":["3d%1:09:00::"],"candidates":[{"index":1,"sense_key":"3-d%1:09:00::","synset_id":"three-d.n.02"},{"index":2,"sense_key":"3-d%1:10:00::","synset_id":"three-d.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"3-d%1:09:00::","call_ids":["semeval2015.d001.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"3-d%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":325,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":86},"cost":{"currency":"USD","total_usd":0.004655,"input_uncached_usd":0.0016250000000000001,"input_cached_usd":0.0,"output_usd":0.00303,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.103297457913868}
{"item_id":"semeval2015.d001.s008.t000","gold_sense_keys":["below%4:02:01::"],"candidates":[{"index":1,"sense_key":"below%4:02:03::","synset_id":"under.r.08"},{"index":2,"sense_key":"below%4:02:06::","synset_id":"downstairs.r.01"},{"index":3,"sense_key":"below%4:02:00::","synset_id":"below.r.02"},{"index":4,"sense_key":"below%4:02:02::","synset_id":"below.r.03"},{"index":5,"sense_key":"below%4:02:01::","synset_id":"below.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"below%4:02:01::","call_ids":["semeval2015.d001.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"below%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":512,"reasoning_output_tokens":496},"cost":{"currency":"USD","total_usd":0.01756,"input_uncached_usd":0.0022,"input_cached_usd":0.0,"output_usd":0.01536,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.451217791996896}
{"item_id":"semeval2015.d001.s008.t001","gold_sense_keys":["find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":2,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":3,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":4,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":5,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":6,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":7,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":8,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":9,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":10,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":11,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":12,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"},{"index":13,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":14,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":15,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":16,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"find%2:40:02::","call_ids":["semeval2015.d001.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"find%2:40:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1064,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.021159999999999998,"input_uncached_usd":0.00532,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.544662334024906}
{"item_id":"semeval2015.d001.s008.t004","gold_sense_keys":["type%2:32:00::"],"candidates":[{"index":1,"sense_key":"type%2:32:00::","synset_id":"type.v.01"},{"index":2,"sense_key":"type%2:31:00::","synset_id":"type.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%2:32:00::","call_ids":["semeval2015.d001.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":349,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.003725,"input_uncached_usd":0.0017450000000000002,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4354372499510646}
{"item_id":"semeval2015.d001.s008.t005","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":2,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":3,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"},{"index":4,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":5,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:24:00::","synset_id":"function.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"function%1:24:00::","call_ids":["semeval2015.d001.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"function%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.005300000000000001,"input_uncached_usd":0.00305,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8330612920690328}
{"item_id":"semeval2015.d001.s008.t006","gold_sense_keys":["do%2:36:01::","do%2:41:01::","do%2:41:04::"],"candidates":[{"index":1,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":2,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":3,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":4,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":5,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":6,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":7,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":8,"sense_key":"do%2:38:00::","synset_id":"do.v.13"},{"index":9,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":10,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":11,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":12,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":13,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"do%2:36:01::","call_ids":["semeval2015.d001.s008.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":869,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":141},"cost":{"currency":"USD","total_usd":0.009055,"input_uncached_usd":0.004345000000000001,"input_cached_usd":0.0,"output_usd":0.00471,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.2833915000082925}
{"item_id":"semeval2015.d001.s008.t007","gold_sense_keys":["calculation%1:04:00::","calculation%1:09:00::"],"candidates":[{"index":1,"sense_key":"calculation%1:04:00::","synset_id":"calculation.n.01"},{"index":2,"sense_key":"calculation%1:09:01::","synset_id":"calculation.n.03"},{"index":3,"sense_key":"calculation%1:09:00::","synset_id":"calculation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"calculation%1:09:00::","call_ids":["semeval2015.d001.s008.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"calculation%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":385,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017765,"input_uncached_usd":0.001925,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.538783750031143}
{"item_id":"semeval2015.d001.s009.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:42:01::","synset_id":"want.v.05"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:35:00::","synset_id":"want.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["semeval2015.d001.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.004185,"input_uncached_usd":0.0023850000000000004,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7248422920238227}
{"item_id":"semeval2015.d001.s009.t001","gold_sense_keys":["understand%2:31:00::","understand%2:31:01::"],"candidates":[{"index":1,"sense_key":"understand%2:31:00::","synset_id":"understand.v.01"},{"index":2,"sense_key":"understand%2:31:04::","synset_id":"sympathize.v.02"},{"index":3,"sense_key":"understand%2:31:03::","synset_id":"understand.v.03"},{"index":4,"sense_key":"understand%2:31:01::","synset_id":"understand.v.02"},{"index":5,"sense_key":"understand%2:31:02::","synset_id":"understand.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"understand%2:31:00::","call_ids":["semeval2015.d001.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"understand%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":114},"cost":{"currency":"USD","total_usd":0.006370000000000001,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.00387,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.057825000025332}
{"item_id":"semeval2015.d001.s009.t002","gold_sense_keys":["work%2:35:02::","work%2:41:03::"],"candidates":[{"index":1,"sense_key":"work%2:41:00::","synset_id":"work.v.02"},{"index":2,"sense_key":"work%2:36:12::","synset_id":"shape.v.02"},{"index":3,"sense_key":"work%2:38:02::","synset_id":"work.v.10"},{"index":4,"sense_key":"work%2:31:13::","synset_id":"solve.v.01"},{"index":5,"sense_key":"work%2:37:10::","synset_id":"work.v.19"},{"index":6,"sense_key":"work%2:30:13::","synset_id":"sour.v.01"},{"index":7,"sense_key":"work%2:41:10::","synset_id":"influence.v.01"},{"index":8,"sense_key":"work%2:38:00::","synset_id":"make.v.36"},{"index":9,"sense_key":"work%2:41:03::","synset_id":"work.v.03"},{"index":10,"sense_key":"work%2:41:12::","synset_id":"work.v.17"},{"index":11,"sense_key":"work%2:35:14::","synset_id":"work.v.21"},{"index":12,"sense_key":"work%2:42:10::","synset_id":"work.v.14"},{"index":13,"sense_key":"work%2:30:14::","synset_id":"ferment.v.03"},{"index":14,"sense_key":"work%2:35:02::","synset_id":"function.v.01"},{"index":15,"sense_key":"work%2:39:10::","synset_id":"work.v.18"},{"index":16,"sense_key":"work%2:35:00::","synset_id":"knead.v.01"},{"index":17,"sense_key":"work%2:36:01::","synset_id":"bring.v.03"},{"index":18,"sense_key":"work%2:41:01::","synset_id":"work.v.12"},{"index":19,"sense_key":"work%2:41:02::","synset_id":"work.v.01"},{"index":20,"sense_key":"work%2:41:11::","synset_id":"work.v.16"},{"index":21,"sense_key":"work%2:36:09::","synset_id":"cultivate.v.02"},{"index":22,"sense_key":"work%2:36:00::","synset_id":"work.v.05"},{"index":23,"sense_key":"work%2:30:10::","synset_id":"work.v.27"},{"index":24,"sense_key":"work%2:34:10::","synset_id":"exploit.v.01"},{"index":25,"sense_key":"work%2:41:04::","synset_id":"work.v.08"},{"index":26,"sense_key":"work%2:41:05::","synset_id":"work.v.09"},{"index":27,"sense_key":"work%2:29:00::","synset_id":"exercise.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"work%2:35:02::","call_ids":["semeval2015.d001.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"work%2:35:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1454,"cached_input_tokens":0,"output_tokens":247,"reasoning_output_tokens":231},"cost":{"currency":"USD","total_usd":0.014680000000000002,"input_uncached_usd":0.00727,"input_cached_usd":0.0,"output_usd":0.00741,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4681140419561416}
{"item_id":"semeval2015.d001.s009.t003","gold_sense_keys":["best%5:00:00:advisable:00"],"candidates":[{"index":1,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":2,"sense_key":"good%5:00:00:healthy:00","synset_id":"good.s.15"},{"index":3,"sense_key":"good%5:00:00:opportune:00","synset_id":"good.s.10"},{"index":4,"sense_key":"dear%5:00:00:close:02","synset_id":"dear.s.02"},{"index":5,"sense_key":"good%5:00:00:complete:00","synset_id":"good.s.07"},{"index":6,"sense_key":"dependable%5:00:00:sound:00","synset_id":"dependable.s.03"},{"index":7,"sense_key":"good%5:00:00:pleasing:00","synset_id":"good.s.13"},{"index":8,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":9,"sense_key":"good%5:00:00:fresh:01","synset_id":"good.s.18"},{"index":10,"sense_key":"good%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":11,"sense_key":"good%5:00:00:nice:00","synset_id":"good.s.04"},{"index":12,"sense_key":"beneficial%5:00:00:advantageous:00","synset_id":"beneficial.s.01"},{"index":13,"sense_key":"good%5:00:00:discriminating:00","synset_id":"good.s.19"},{"index":14,"sense_key":"adept%5:00:00:skilled:00","synset_id":"adept.s.01"},{"index":15,"sense_key":"estimable%5:00:00:reputable:00","synset_id":"estimable.s.01"},{"index":16,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":17,"sense_key":"good%5:00:00:righteous:00","synset_id":"good.s.05"},{"index":18,"sense_key":"good%3:00:02::","synset_id":"good.a.03"},{"index":19,"sense_key":"best%3:00:00::","synset_id":"best.a.01"},{"index":20,"sense_key":"best%5:00:00:advisable:00","synset_id":"better.s.01"},{"index":21,"sense_key":"good%3:00:01::","synset_id":"good.a.01"},{"index":22,"sense_key":"good%5:00:00:healthful:00","synset_id":"good.s.16"},{"index":23,"sense_key":"good%5:00:00:genuine:00","synset_id":"good.s.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":20,"chosen_sense_key":"best%5:00:00:advisable:00","call_ids":["semeval2015.d001.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":20,"predicted_sense_key":"best%5:00:00:advisable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1319,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.022435,"input_uncached_usd":0.006595,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.712420458905399}
{"item_id":"semeval2015.d001.s009.t004","gold_sense_keys":["thing%1:04:00::"],"candidates":[{"index":1,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":4,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":5,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":6,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":7,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"},{"index":8,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":9,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":10,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":11,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":12,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"thing%1:04:00::","call_ids":["semeval2015.d001.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"thing%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":101},"cost":{"currency":"USD","total_usd":0.006895,"input_uncached_usd":0.0033850000000000004,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6673204170074314}
{"item_id":"semeval2015.d001.s009.t005","gold_sense_keys":["do%2:36:01::"],"candidates":[{"index":1,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":2,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":3,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":4,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":5,"sense_key":"do%2:38:00::","synset_id":"do.v.13"},{"index":6,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":7,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":8,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":11,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":12,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":13,"sense_key":"do%2:41:03::","synset_id":"do.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"do%2:36:01::","call_ids":["semeval2015.d001.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":877,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020195,"input_uncached_usd":0.004385,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.889331083046272}
{"item_id":"semeval2015.d001.s009.t006","gold_sense_keys":["realize%2:31:01::"],"candidates":[{"index":1,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"},{"index":2,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":3,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"},{"index":4,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"},{"index":5,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"},{"index":6,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"realize%2:31:01::","call_ids":["semeval2015.d001.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"realize%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":576,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01872,"input_uncached_usd":0.00288,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.238813291070983}
{"item_id":"semeval2015.d001.s009.t008","gold_sense_keys":["convert%2:30:00::","convert%2:30:02::","convert%2:30:03::"],"candidates":[{"index":1,"sense_key":"convert%2:30:01::","synset_id":"convert.v.03"},{"index":2,"sense_key":"convert%2:30:07::","synset_id":"convert.v.05"},{"index":3,"sense_key":"convert%2:30:02::","synset_id":"change.v.06"},{"index":4,"sense_key":"convert%2:30:04::","synset_id":"convert.v.11"},{"index":5,"sense_key":"convert%2:32:00::","synset_id":"convert.v.09"},{"index":6,"sense_key":"convert%2:33:02::","synset_id":"convert.v.06"},{"index":7,"sense_key":"convert%2:30:00::","synset_id":"convert.v.01"},{"index":8,"sense_key":"convert%2:30:09::","synset_id":"commute.v.04"},{"index":9,"sense_key":"convert%2:33:01::","synset_id":"convert.v.07"},{"index":10,"sense_key":"convert%2:33:00::","synset_id":"convert.v.08"},{"index":11,"sense_key":"convert%2:30:03::","synset_id":"convert.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"convert%2:30:02::","call_ids":["semeval2015.d001.s009.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"convert%2:30:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":714,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01941,"input_uncached_usd":0.0035700000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.431439582956955}
{"item_id":"semeval2015.d001.s009.t010","gold_sense_keys":["necessary%3:00:00::"],"candidates":[{"index":1,"sense_key":"necessary%5:00:00:inevitable:00","synset_id":"necessary.s.01"},{"index":2,"sense_key":"necessary%3:00:00::","synset_id":"necessary.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"necessary%3:00:00::","call_ids":["semeval2015.d001.s009.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"necessary%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":338,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00346,"input_uncached_usd":0.00169,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.906463082996197}
{"item_id":"semeval2015.d001.s009.t011","gold_sense_keys":["know%2:31:15::"],"candidates":[{"index":1,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":2,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":3,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":4,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":5,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":6,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":7,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":8,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":9,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":10,"sense_key":"know%2:31:14::","synset_id":"know.v.11"},{"index":11,"sense_key":"know%2:31:03::","synset_id":"know.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"know%2:31:15::","call_ids":["semeval2015.d001.s009.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"know%2:31:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":795,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":90},"cost":{"currency":"USD","total_usd":0.007125,"input_uncached_usd":0.003975,"input_cached_usd":0.0,"output_usd":0.00315,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5869563749292865}
{"item_id":"semeval2015.d001.s010.t000","gold_sense_keys":["base%2:31:00::"],"candidates":[{"index":1,"sense_key":"base%2:31:00::","synset_id":"establish.v.08"},{"index":2,"sense_key":"base%2:42:04::","synset_id":"base.v.02"},{"index":3,"sense_key":"base%2:34:00::","synset_id":"free-base.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"base%2:31:00::","call_ids":["semeval2015.d001.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"base%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":392,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.0037600000000000003,"input_uncached_usd":0.0019600000000000004,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7190778340445831}
{"item_id":"semeval2015.d001.s010.t004","gold_sense_keys":["hard%3:00:06::"],"candidates":[{"index":1,"sense_key":"hard%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":2,"sense_key":"hard%5:00:01:effortful:00","synset_id":"arduous.s.01"},{"index":3,"sense_key":"hard%3:00:06::","synset_id":"difficult.a.01"},{"index":4,"sense_key":"hard%3:00:03::","synset_id":"hard.a.08"},{"index":5,"sense_key":"hard%3:00:05::","synset_id":"hard.a.07"},{"index":6,"sense_key":"hard%5:00:00:indulgent:00","synset_id":"intemperate.s.01"},{"index":7,"sense_key":"hard%3:00:01::","synset_id":"hard.a.03"},{"index":8,"sense_key":"hard%5:00:00:alcoholic:00","synset_id":"hard.s.04"},{"index":9,"sense_key":"hard%5:00:00:stale:00","synset_id":"hard.s.06"},{"index":10,"sense_key":"hard%3:00:00::","synset_id":"unvoiced.a.01"},{"index":11,"sense_key":"hard%3:00:02::","synset_id":"hard.a.02"},{"index":12,"sense_key":"hard%5:00:00:bad:00","synset_id":"hard.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"hard%3:00:06::","call_ids":["semeval2015.d001.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"hard%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":805,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.005645,"input_uncached_usd":0.004025,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6287837909767404}
{"item_id":"semeval2015.d001.s011.t001","gold_sense_keys":["closely%4:02:02::"],"candidates":[{"index":1,"sense_key":"closely%4:02:02::","synset_id":"close.r.02"},{"index":2,"sense_key":"closely%4:02:01::","synset_id":"closely.r.03"},{"index":3,"sense_key":"closely%4:02:00::","synset_id":"closely.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"closely%4:02:02::","call_ids":["semeval2015.d001.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"closely%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.00453,"input_uncached_usd":0.00201,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2158185420557857}
{"item_id":"semeval2015.d001.s011.t002","gold_sense_keys":["look%2:39:01::"],"candidates":[{"index":1,"sense_key":"look%2:39:02::","synset_id":"search.v.02"},{"index":2,"sense_key":"look%2:31:02::","synset_id":"count.v.08"},{"index":3,"sense_key":"look%2:29:00::","synset_id":"look.v.03"},{"index":4,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":5,"sense_key":"look%2:39:00::","synset_id":"look.v.01"},{"index":6,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":7,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":8,"sense_key":"look%2:42:00::","synset_id":"front.v.01"},{"index":9,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":10,"sense_key":"look%2:39:01::","synset_id":"look.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"look%2:39:01::","call_ids":["semeval2015.d001.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"look%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.006145,"input_uncached_usd":0.003625,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1876644160365686}
{"item_id":"semeval2015.d001.s011.t003","gold_sense_keys":["very%4:02:00::"],"candidates":[{"index":1,"sense_key":"very%4:02:01::","synset_id":"very.r.02"},{"index":2,"sense_key":"very%4:02:00::","synset_id":"very.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"very%4:02:00::","call_ids":["semeval2015.d001.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"very%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":379,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.0040550000000000004,"input_uncached_usd":0.0018950000000000002,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":78.9110980409896}
{"item_id":"semeval2015.d001.s011.t004","gold_sense_keys":["similar%3:00:00::","similar%3:00:02::","similar%3:00:04::"],"candidates":[{"index":1,"sense_key":"similar%3:00:02::","synset_id":"like.a.01"},{"index":2,"sense_key":"similar%5:00:00:synonymous:00","synset_id":"similar.s.01"},{"index":3,"sense_key":"similar%5:00:00:replaceable:00","synset_id":"exchangeable.s.01"},{"index":4,"sense_key":"similar%3:00:00::","synset_id":"similar.a.01"},{"index":5,"sense_key":"similar%3:00:04::","synset_id":"alike.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"similar%3:00:00::","call_ids":["semeval2015.d001.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"similar%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":2600,"reasoning_output_tokens":2559},"cost":{"currency":"USD","total_usd":0.080515,"input_uncached_usd":0.0025150000000000003,"input_cached_usd":0.0,"output_usd":0.078,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":40.74493662500754}
{"item_id":"semeval2015.d001.s012.t000","gold_sense_keys":["addition%1:04:01::"],"candidates":[{"index":1,"sense_key":"addition%1:23:00::","synset_id":"addition.n.03"},{"index":2,"sense_key":"addition%1:21:00::","synset_id":"accession.n.03"},{"index":3,"sense_key":"addition%1:15:00::","synset_id":"addition.n.05"},{"index":4,"sense_key":"addition%1:04:02::","synset_id":"addition.n.02"},{"index":5,"sense_key":"addition%1:06:00::","synset_id":"addition.n.01"},{"index":6,"sense_key":"addition%1:04:01::","synset_id":"summation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"addition%1:04:01::","call_ids":["semeval2015.d001.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"addition%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.0049,"input_uncached_usd":0.00289,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5411234999774024}
{"item_id":"semeval2015.d001.s012.t001","gold_sense_keys":["subtraction%1:04:01::"],"candidates":[{"index":1,"sense_key":"subtraction%1:04:01::","synset_id":"subtraction.n.01"},{"index":2,"sense_key":"subtraction%1:04:00::","synset_id":"subtraction.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"subtraction%1:04:01::","call_ids":["semeval2015.d001.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"subtraction%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.00351,"input_uncached_usd":0.0019500000000000001,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3607537089847028}
{"item_id":"semeval2015.d001.s012.t002","gold_sense_keys":["multiplication%1:04:00::"],"candidates":[{"index":1,"sense_key":"multiplication%1:22:00::","synset_id":"multiplication.n.02"},{"index":2,"sense_key":"multiplication%1:04:00::","synset_id":"multiplication.n.03"},{"index":3,"sense_key":"multiplication%1:04:01::","synset_id":"generation.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"multiplication%1:04:00::","call_ids":["semeval2015.d001.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"multiplication%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.0038900000000000002,"input_uncached_usd":0.0021200000000000004,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4014277920359746}
{"item_id":"semeval2015.d001.s012.t003","gold_sense_keys":["division%1:04:01::"],"candidates":[{"index":1,"sense_key":"division%1:14:00::","synset_id":"division.n.01"},{"index":2,"sense_key":"division%1:14:03::","synset_id":"division.n.04"},{"index":3,"sense_key":"division%1:10:00::","synset_id":"division.n.05"},{"index":4,"sense_key":"division%1:14:02::","synset_id":"division.n.09"},{"index":5,"sense_key":"division%1:04:02::","synset_id":"division.n.12"},{"index":6,"sense_key":"division%1:14:06::","synset_id":"class.n.05"},{"index":7,"sense_key":"division%1:09:00::","synset_id":"part.n.09"},{"index":8,"sense_key":"division%1:04:00::","synset_id":"division.n.03"},{"index":9,"sense_key":"division%1:14:04::","synset_id":"division.n.08"},{"index":10,"sense_key":"division%1:04:01::","synset_id":"division.n.11"},{"index":11,"sense_key":"division%1:14:05::","synset_id":"division.n.07"},{"index":12,"sense_key":"division%1:14:01::","synset_id":"division.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"division%1:04:01::","call_ids":["semeval2015.d001.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"division%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.0052,"input_uncached_usd":0.00355,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3556709579424933}
{"item_id":"semeval2015.d001.s013.t000","gold_sense_keys":["power%1:10:00::"],"candidates":[{"index":1,"sense_key":"power%1:14:00::","synset_id":"world_power.n.01"},{"index":2,"sense_key":"power%1:07:00::","synset_id":"power.n.01"},{"index":3,"sense_key":"power%1:18:01::","synset_id":"baron.n.03"},{"index":4,"sense_key":"power%1:19:00::","synset_id":"power.n.02"},{"index":5,"sense_key":"power%1:10:00::","synset_id":"exponent.n.03"},{"index":6,"sense_key":"power%1:26:00::","synset_id":"office.n.04"},{"index":7,"sense_key":"power%1:09:01::","synset_id":"ability.n.02"},{"index":8,"sense_key":"power%1:18:00::","synset_id":"power.n.05"},{"index":9,"sense_key":"power%1:07:02::","synset_id":"might.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"power%1:10:00::","call_ids":["semeval2015.d001.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"power%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.00635,"input_uncached_usd":0.0032600000000000003,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.457220792071894}
{"item_id":"semeval2015.d001.s013.t001","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":2,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":3,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":4,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":5,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":6,"sense_key":"use%2:34:00::","synset_id":"use.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.0048200000000000005,"input_uncached_usd":0.0027800000000000004,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.704287042026408}
{"item_id":"semeval2015.d001.s014.t001","gold_sense_keys":["possible%3:00:00::"],"candidates":[{"index":1,"sense_key":"possible%3:00:04::","synset_id":"potential.a.01"},{"index":2,"sense_key":"possible%3:00:00::","synset_id":"possible.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"possible%3:00:00::","call_ids":["semeval2015.d001.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"possible%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":336,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.00477,"input_uncached_usd":0.00168,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.990513916942291}
{"item_id":"semeval2015.d001.s014.t002","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":2,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":3,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":4,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":5,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":6,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004775,"input_uncached_usd":0.002735,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9810188339324668}
{"item_id":"semeval2015.d001.s015.t000","gold_sense_keys":["power%1:10:00::"],"candidates":[{"index":1,"sense_key":"power%1:07:00::","synset_id":"power.n.01"},{"index":2,"sense_key":"power%1:07:02::","synset_id":"might.n.01"},{"index":3,"sense_key":"power%1:26:00::","synset_id":"office.n.04"},{"index":4,"sense_key":"power%1:18:00::","synset_id":"power.n.05"},{"index":5,"sense_key":"power%1:18:01::","synset_id":"baron.n.03"},{"index":6,"sense_key":"power%1:09:01::","synset_id":"ability.n.02"},{"index":7,"sense_key":"power%1:10:00::","synset_id":"exponent.n.03"},{"index":8,"sense_key":"power%1:14:00::","synset_id":"world_power.n.01"},{"index":9,"sense_key":"power%1:19:00::","synset_id":"power.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"power%1:10:00::","call_ids":["semeval2015.d001.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"power%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.005695,"input_uncached_usd":0.003115,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4453370000701398}
{"item_id":"semeval2015.d001.s015.t001","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":2,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":3,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":4,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":5,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":6,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":7,"sense_key":"way%1:21:00::","synset_id":"way.n.12"},{"index":8,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":9,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":10,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":11,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":12,"sense_key":"way%1:26:00::","synset_id":"way.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"way%1:04:01::","call_ids":["semeval2015.d001.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":715,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":129},"cost":{"currency":"USD","total_usd":0.007925000000000001,"input_uncached_usd":0.0035750000000000005,"input_cached_usd":0.0,"output_usd":0.00435,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8551696659997106}
{"item_id":"semeval2015.d001.s015.t003","gold_sense_keys":["root%1:23:00::"],"candidates":[{"index":1,"sense_key":"root%1:23:00::","synset_id":"root.n.04"},{"index":2,"sense_key":"root%1:10:00::","synset_id":"root.n.03"},{"index":3,"sense_key":"root%1:14:00::","synset_id":"solution.n.04"},{"index":4,"sense_key":"root%1:10:01::","synset_id":"etymon.n.01"},{"index":5,"sense_key":"root%1:08:00::","synset_id":"root.n.08"},{"index":6,"sense_key":"root%1:15:00::","synset_id":"beginning.n.04"},{"index":7,"sense_key":"root%1:20:00::","synset_id":"root.n.01"},{"index":8,"sense_key":"root%1:18:00::","synset_id":"ancestor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"root%1:23:00::","call_ids":["semeval2015.d001.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"root%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":114},"cost":{"currency":"USD","total_usd":0.006815,"input_uncached_usd":0.002945,"input_cached_usd":0.0,"output_usd":0.00387,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.328291874960996}
{"item_id":"semeval2015.d001.s015.t004","gold_sense_keys":["too%4:02:01::"],"candidates":[{"index":1,"sense_key":"too%4:02:01::","synset_id":"besides.r.02"},{"index":2,"sense_key":"too%4:02:00::","synset_id":"excessively.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"too%4:02:01::","call_ids":["semeval2015.d001.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"too%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":324,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.0035700000000000003,"input_uncached_usd":0.0016200000000000001,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6249377500498667}
{"item_id":"semeval2015.d001.s016.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":2,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":5,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":6,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":7,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":8,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":9,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":10,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":11,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"way%1:04:01::","call_ids":["semeval2015.d001.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":227,"reasoning_output_tokens":211},"cost":{"currency":"USD","total_usd":0.010345000000000002,"input_uncached_usd":0.0035350000000000004,"input_cached_usd":0.0,"output_usd":0.00681,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.329254208016209}
{"item_id":"semeval2015.d001.s016.t001","gold_sense_keys":["specify%2:32:02::"],"candidates":[{"index":1,"sense_key":"specify%2:32:03::","synset_id":"stipulate.v.01"},{"index":2,"sense_key":"specify%2:31:05::","synset_id":"assign.v.04"},{"index":3,"sense_key":"specify%2:42:00::","synset_id":"specify.v.03"},{"index":4,"sense_key":"specify%2:31:02::","synset_id":"pin_down.v.01"},{"index":5,"sense_key":"specify%2:32:02::","synset_id":"specify.v.02"},{"index":6,"sense_key":"specify%2:32:00::","synset_id":"specify.v.04"},{"index":7,"sense_key":"specify%2:31:00::","synset_id":"intend.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"specify%2:32:02::","call_ids":["semeval2015.d001.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"specify%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":352,"reasoning_output_tokens":336},"cost":{"currency":"USD","total_usd":0.013455,"input_uncached_usd":0.0028950000000000004,"input_cached_usd":0.0,"output_usd":0.01056,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.466189708909951}
{"item_id":"semeval2015.d001.s016.t003","gold_sense_keys":["variable%1:10:00::"],"candidates":[{"index":1,"sense_key":"variable%1:17:01::","synset_id":"variable_star.n.01"},{"index":2,"sense_key":"variable%1:10:00::","synset_id":"variable.n.04"},{"index":3,"sense_key":"variable%1:17:00::","synset_id":"variable.n.01"},{"index":4,"sense_key":"variable%1:09:00::","synset_id":"variable.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"variable%1:10:00::","call_ids":["semeval2015.d001.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"variable%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":374,"cached_input_tokens":0,"output_tokens":220,"reasoning_output_tokens":204},"cost":{"currency":"USD","total_usd":0.008470000000000002,"input_uncached_usd":0.0018700000000000001,"input_cached_usd":0.0,"output_usd":0.0066,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.428359416080639}
{"item_id":"semeval2015.d001.s016.t004","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":4,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":5,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"},{"index":6,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":7,"sense_key":"function%1:14:00::","synset_id":"function.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"function%1:24:00::","call_ids":["semeval2015.d001.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"function%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":135},"cost":{"currency":"USD","total_usd":0.007365,"input_uncached_usd":0.0028650000000000004,"input_cached_usd":0.0,"output_usd":0.0045000000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3334608749719337}
{"item_id":"semeval2015.d001.s017.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":4,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":5,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":6,"sense_key":"use%2:41:14::","synset_id":"use.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.00461,"input_uncached_usd":0.0026000000000000003,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7804733749944717}
{"item_id":"semeval2015.d001.s017.t001","gold_sense_keys":["specify%2:32:02::"],"candidates":[{"index":1,"sense_key":"specify%2:32:03::","synset_id":"stipulate.v.01"},{"index":2,"sense_key":"specify%2:31:05::","synset_id":"assign.v.04"},{"index":3,"sense_key":"specify%2:32:02::","synset_id":"specify.v.02"},{"index":4,"sense_key":"specify%2:31:00::","synset_id":"intend.v.02"},{"index":5,"sense_key":"specify%2:42:00::","synset_id":"specify.v.03"},{"index":6,"sense_key":"specify%2:32:00::","synset_id":"specify.v.04"},{"index":7,"sense_key":"specify%2:31:02::","synset_id":"pin_down.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"specify%2:32:02::","call_ids":["semeval2015.d001.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"specify%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":581,"cached_input_tokens":0,"output_tokens":170,"reasoning_output_tokens":154},"cost":{"currency":"USD","total_usd":0.008005,"input_uncached_usd":0.002905,"input_cached_usd":0.0,"output_usd":0.0051,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9357493340503424}
{"item_id":"semeval2015.d001.s017.t002","gold_sense_keys":["high%3:00:02::"],"candidates":[{"index":1,"sense_key":"high%5:00:00:intoxicated:00","synset_id":"high.s.04"},{"index":2,"sense_key":"high%3:00:03::","synset_id":"high.a.04"},{"index":3,"sense_key":"high%5:00:00:elated:00","synset_id":"high.s.02"},{"index":4,"sense_key":"high%5:00:00:superior:01","synset_id":"eminent.s.01"},{"index":5,"sense_key":"high%5:00:00:malodorous:00","synset_id":"gamey.s.02"},{"index":6,"sense_key":"high%3:00:01::","synset_id":"high.a.02"},{"index":7,"sense_key":"high%3:00:02::","synset_id":"high.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"high%3:00:02::","call_ids":["semeval2015.d001.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"high%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01842,"input_uncached_usd":0.0025800000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.339101209072396}
{"item_id":"semeval2015.d001.s017.t003","gold_sense_keys":["priority%1:26:00::"],"candidates":[{"index":1,"sense_key":"priority%1:07:00::","synset_id":"priority.n.02"},{"index":2,"sense_key":"priority%1:26:00::","synset_id":"precedence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"priority%1:26:00::","call_ids":["semeval2015.d001.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"priority%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":327,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":93},"cost":{"currency":"USD","total_usd":0.004905,"input_uncached_usd":0.0016350000000000002,"input_cached_usd":0.0,"output_usd":0.00327,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.348610249930061}
{"item_id":"semeval2015.d001.s018.t001","gold_sense_keys":["find%2:32:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":2,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":3,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":4,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":5,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":6,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":7,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":8,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":9,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":10,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":13,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":14,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":15,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"},{"index":16,"sense_key":"find%2:39:05::","synset_id":"find.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"find%2:39:02::","call_ids":["semeval2015.d001.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"find%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1044,"cached_input_tokens":0,"output_tokens":199,"reasoning_output_tokens":183},"cost":{"currency":"USD","total_usd":0.011190000000000002,"input_uncached_usd":0.005220000000000001,"input_cached_usd":0.0,"output_usd":0.0059700000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.469163208035752}
{"item_id":"semeval2015.d001.s018.t002","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"},{"index":2,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":3,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":4,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":5,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":6,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":7,"sense_key":"function%1:07:00::","synset_id":"function.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"function%1:10:00::","call_ids":["semeval2015.d001.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"function%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01878,"input_uncached_usd":0.0029400000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":17.873131375061348}
{"item_id":"semeval2015.d001.s018.t003","gold_sense_keys":["check%2:32:10::"],"candidates":[{"index":1,"sense_key":"check%2:30:09::","synset_id":"check.v.24"},{"index":2,"sense_key":"check%2:38:15::","synset_id":"check.v.05"},{"index":3,"sense_key":"check%2:31:01::","synset_id":"check.v.06"},{"index":4,"sense_key":"check%2:38:12::","synset_id":"check.v.15"},{"index":5,"sense_key":"check%2:32:07::","synset_id":"check.v.20"},{"index":6,"sense_key":"check%2:40:00::","synset_id":"check.v.13"},{"index":7,"sense_key":"check%2:30:10::","synset_id":"check.v.23"},{"index":8,"sense_key":"check%2:42:08::","synset_id":"match.v.01"},{"index":9,"sense_key":"check%2:41:08::","synset_id":"control.v.02"},{"index":10,"sense_key":"check%2:32:10::","synset_id":"determine.v.08"},{"index":11,"sense_key":"check%2:40:13::","synset_id":"check.v.12"},{"index":12,"sense_key":"check%2:31:00::","synset_id":"check.v.01"},{"index":13,"sense_key":"check%2:41:14::","synset_id":"check.v.10"},{"index":14,"sense_key":"check%2:33:06::","synset_id":"check.v.18"},{"index":15,"sense_key":"check%2:33:02::","synset_id":"check.v.17"},{"index":16,"sense_key":"check%2:30:03::","synset_id":"crack.v.01"},{"index":17,"sense_key":"check%2:38:14::","synset_id":"check.v.14"},{"index":18,"sense_key":"check%2:31:12::","synset_id":"check.v.22"},{"index":19,"sense_key":"check%2:41:02::","synset_id":"discipline.v.01"},{"index":20,"sense_key":"check%2:42:09::","synset_id":"check.v.08"},{"index":21,"sense_key":"check%2:36:00::","synset_id":"check.v.16"},{"index":22,"sense_key":"check%2:30:02::","synset_id":"check.v.07"},{"index":23,"sense_key":"check%2:33:00::","synset_id":"check.v.19"},{"index":24,"sense_key":"check%2:31:02::","synset_id":"see.v.10"},{"index":25,"sense_key":"check%2:41:04::","synset_id":"check.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"check%2:32:10::","call_ids":["semeval2015.d001.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"check%2:32:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1332,"cached_input_tokens":0,"output_tokens":161,"reasoning_output_tokens":145},"cost":{"currency":"USD","total_usd":0.01149,"input_uncached_usd":0.00666,"input_cached_usd":0.0,"output_usd":0.00483,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9883567499928176}
{"item_id":"semeval2015.d001.s018.t004","gold_sense_keys":["operator%1:24:00::"],"candidates":[{"index":1,"sense_key":"operator%1:18:02::","synset_id":"hustler.n.02"},{"index":2,"sense_key":"operator%1:24:00::","synset_id":"operator.n.01"},{"index":3,"sense_key":"operator%1:18:00::","synset_id":"operator.n.02"},{"index":4,"sense_key":"operator%1:18:03::","synset_id":"operator.n.05"},{"index":5,"sense_key":"operator%1:18:01::","synset_id":"operator.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"operator%1:24:00::","call_ids":["semeval2015.d001.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"operator%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.00405,"input_uncached_usd":0.0021000000000000003,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8495997079880908}
{"item_id":"semeval2015.d001.s019.t000","gold_sense_keys":["treat%2:31:00::"],"candidates":[{"index":1,"sense_key":"treat%2:31:00::","synset_id":"treat.v.08"},{"index":2,"sense_key":"treat%2:41:00::","synset_id":"treat.v.01"},{"index":3,"sense_key":"treat%2:32:03::","synset_id":"treat.v.07"},{"index":4,"sense_key":"treat%2:29:00::","synset_id":"treat.v.03"},{"index":5,"sense_key":"treat%2:32:00::","synset_id":"cover.v.05"},{"index":6,"sense_key":"treat%2:30:01::","synset_id":"process.v.01"},{"index":7,"sense_key":"treat%2:40:00::","synset_id":"treat.v.05"},{"index":8,"sense_key":"treat%2:34:00::","synset_id":"regale.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treat%2:31:00::","call_ids":["semeval2015.d001.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treat%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.005805,"input_uncached_usd":0.003105,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8317579999566078}
{"item_id":"semeval2015.d001.s019.t001","gold_sense_keys":["operator%1:24:00::"],"candidates":[{"index":1,"sense_key":"operator%1:24:00::","synset_id":"operator.n.01"},{"index":2,"sense_key":"operator%1:18:02::","synset_id":"hustler.n.02"},{"index":3,"sense_key":"operator%1:18:01::","synset_id":"operator.n.03"},{"index":4,"sense_key":"operator%1:18:00::","synset_id":"operator.n.02"},{"index":5,"sense_key":"operator%1:18:03::","synset_id":"operator.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"operator%1:24:00::","call_ids":["semeval2015.d001.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"operator%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.003745,"input_uncached_usd":0.002095,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.302893542102538}
{"item_id":"semeval2015.d001.s019.t003","gold_sense_keys":["treat%2:31:00::"],"candidates":[{"index":1,"sense_key":"treat%2:41:00::","synset_id":"treat.v.01"},{"index":2,"sense_key":"treat%2:32:03::","synset_id":"treat.v.07"},{"index":3,"sense_key":"treat%2:34:00::","synset_id":"regale.v.01"},{"index":4,"sense_key":"treat%2:40:00::","synset_id":"treat.v.05"},{"index":5,"sense_key":"treat%2:31:00::","synset_id":"treat.v.08"},{"index":6,"sense_key":"treat%2:32:00::","synset_id":"cover.v.05"},{"index":7,"sense_key":"treat%2:29:00::","synset_id":"treat.v.03"},{"index":8,"sense_key":"treat%2:30:01::","synset_id":"process.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"treat%2:31:00::","call_ids":["semeval2015.d001.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"treat%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":198,"reasoning_output_tokens":182},"cost":{"currency":"USD","total_usd":0.009045000000000001,"input_uncached_usd":0.003105,"input_cached_usd":0.0,"output_usd":0.00594,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9202292090049013}
{"item_id":"semeval2015.d001.s019.t004","gold_sense_keys":["user%1:18:00::"],"candidates":[{"index":1,"sense_key":"user%1:18:02::","synset_id":"drug_user.n.01"},{"index":2,"sense_key":"user%1:18:00::","synset_id":"user.n.01"},{"index":3,"sense_key":"user%1:18:01::","synset_id":"exploiter.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"user%1:18:00::","call_ids":["semeval2015.d001.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"user%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":343,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.004865,"input_uncached_usd":0.0017150000000000002,"input_cached_usd":0.0,"output_usd":0.00315,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.569243583944626}
{"item_id":"semeval2015.d001.s019.t005","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":2,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"},{"index":3,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":4,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":5,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":6,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":7,"sense_key":"function%1:04:00::","synset_id":"function.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"function%1:10:00::","call_ids":["semeval2015.d001.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"function%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":646,"reasoning_output_tokens":623},"cost":{"currency":"USD","total_usd":0.022320000000000003,"input_uncached_usd":0.0029400000000000003,"input_cached_usd":0.0,"output_usd":0.01938,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.234222624916583}
{"item_id":"semeval2015.d001.s020.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":2,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":3,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":4,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:34:02::","synset_id":"use.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004675,"input_uncached_usd":0.002635,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9946659170091152}
{"item_id":"semeval2015.d001.s020.t001","gold_sense_keys":["define%2:32:01::"],"candidates":[{"index":1,"sense_key":"define%2:32:01::","synset_id":"specify.v.02"},{"index":2,"sense_key":"define%2:42:00::","synset_id":"specify.v.03"},{"index":3,"sense_key":"define%2:42:01::","synset_id":"define.v.03"},{"index":4,"sense_key":"define%2:32:00::","synset_id":"define.v.02"},{"index":5,"sense_key":"define%2:42:03::","synset_id":"define.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"define%2:32:01::","call_ids":["semeval2015.d001.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"define%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":96},"cost":{"currency":"USD","total_usd":0.005515000000000001,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9524540830170736}
{"item_id":"semeval2015.d001.s020.t002","gold_sense_keys":["value%1:09:00::"],"candidates":[{"index":1,"sense_key":"value%1:09:00::","synset_id":"value.n.01"},{"index":2,"sense_key":"value%1:28:00::","synset_id":"value.n.05"},{"index":3,"sense_key":"value%1:07:01::","synset_id":"value.n.04"},{"index":4,"sense_key":"value%1:09:01::","synset_id":"value.n.06"},{"index":5,"sense_key":"value%1:07:00::","synset_id":"value.n.02"},{"index":6,"sense_key":"value%1:21:00::","synset_id":"value.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"value%1:09:00::","call_ids":["semeval2015.d001.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"value%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":235,"reasoning_output_tokens":219},"cost":{"currency":"USD","total_usd":0.009535000000000002,"input_uncached_usd":0.0024850000000000002,"input_cached_usd":0.0,"output_usd":0.00705,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5063020829111338}
{"item_id":"semeval2015.d001.s022.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:21:00::","synset_id":"way.n.12"},{"index":2,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":3,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":4,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":5,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":6,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":7,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":8,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":9,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":10,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":11,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":12,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"way%1:04:01::","call_ids":["semeval2015.d001.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":740,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019540000000000002,"input_uncached_usd":0.0037,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.85591679206118}
{"item_id":"semeval2015.d001.s022.t001","gold_sense_keys":["define%2:32:01::"],"candidates":[{"index":1,"sense_key":"define%2:42:03::","synset_id":"define.v.04"},{"index":2,"sense_key":"define%2:42:01::","synset_id":"define.v.03"},{"index":3,"sense_key":"define%2:32:01::","synset_id":"specify.v.02"},{"index":4,"sense_key":"define%2:32:00::","synset_id":"define.v.02"},{"index":5,"sense_key":"define%2:42:00::","synset_id":"specify.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"define%2:32:01::","call_ids":["semeval2015.d001.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"define%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":90},"cost":{"currency":"USD","total_usd":0.005485,"input_uncached_usd":0.002305,"input_cached_usd":0.0,"output_usd":0.00318,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.160225333995186}
{"item_id":"semeval2015.d001.s022.t002","gold_sense_keys":["conditional%3:00:00::"],"candidates":[{"index":1,"sense_key":"conditional%5:00:00:qualified:02","synset_id":"conditional.s.01"},{"index":2,"sense_key":"conditional%3:00:00::","synset_id":"conditional.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"conditional%3:00:00::","call_ids":["semeval2015.d001.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"conditional%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":330,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.00411,"input_uncached_usd":0.0016500000000000002,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7904147499939427}
{"item_id":"semeval2015.d001.s022.t003","gold_sense_keys":["operation%1:04:01::"],"candidates":[{"index":1,"sense_key":"operation%1:04:06::","synset_id":"operation.n.02"},{"index":2,"sense_key":"operation%1:22:01::","synset_id":"operation.n.08"},{"index":3,"sense_key":"operation%1:04:05::","synset_id":"operation.n.07"},{"index":4,"sense_key":"operation%1:04:02::","synset_id":"operation.n.05"},{"index":5,"sense_key":"operation%1:04:03::","synset_id":"operation.n.03"},{"index":6,"sense_key":"operation%1:04:01::","synset_id":"mathematical_process.n.01"},{"index":7,"sense_key":"operation%1:04:07::","synset_id":"operation.n.11"},{"index":8,"sense_key":"operation%1:09:00::","synset_id":"process.n.02"},{"index":9,"sense_key":"operation%1:26:00::","synset_id":"operation.n.01"},{"index":10,"sense_key":"operation%1:04:00::","synset_id":"operation.n.06"},{"index":11,"sense_key":"operation%1:22:00::","synset_id":"operation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"operation%1:04:01::","call_ids":["semeval2015.d001.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"operation%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":829,"cached_input_tokens":0,"output_tokens":327,"reasoning_output_tokens":311},"cost":{"currency":"USD","total_usd":0.013954999999999999,"input_uncached_usd":0.004145,"input_cached_usd":0.0,"output_usd":0.009810000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.2714384170249104}
{"item_id":"semeval2015.d001.s023.t000","gold_sense_keys":["introduce%2:30:03::","introduce%2:35:00::"],"candidates":[{"index":1,"sense_key":"introduce%2:32:02::","synset_id":"introduce.v.07"},{"index":2,"sense_key":"introduce%2:30:03::","synset_id":"insert.v.02"},{"index":3,"sense_key":"introduce%2:32:00::","synset_id":"introduce.v.01"},{"index":4,"sense_key":"introduce%2:35:00::","synset_id":"insert.v.01"},{"index":5,"sense_key":"introduce%2:36:01::","synset_id":"introduce.v.05"},{"index":6,"sense_key":"introduce%2:38:00::","synset_id":"bring_in.v.01"},{"index":7,"sense_key":"introduce%2:30:00::","synset_id":"inaugurate.v.03"},{"index":8,"sense_key":"introduce%2:32:01::","synset_id":"precede.v.05"},{"index":9,"sense_key":"introduce%2:31:00::","synset_id":"introduce.v.08"},{"index":10,"sense_key":"introduce%2:36:00::","synset_id":"introduce.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"introduce%2:35:00::","call_ids":["semeval2015.d001.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"introduce%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":706,"cached_input_tokens":0,"output_tokens":2335,"reasoning_output_tokens":2296},"cost":{"currency":"USD","total_usd":0.07358,"input_uncached_usd":0.00353,"input_cached_usd":0.0,"output_usd":0.07005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":34.99206654110458}
{"item_id":"semeval2015.d001.s023.t002","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":2,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":3,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":4,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":5,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":6,"sense_key":"use%2:34:01::","synset_id":"use.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":469,"reasoning_output_tokens":453},"cost":{"currency":"USD","total_usd":0.016805,"input_uncached_usd":0.002735,"input_cached_usd":0.0,"output_usd":0.014070000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.493965792004019}
{"item_id":"semeval2015.d001.s023.t005","gold_sense_keys":["true%3:00:00::"],"candidates":[{"index":1,"sense_key":"true%3:00:04::","synset_id":"truthful.a.01"},{"index":2,"sense_key":"true%5:00:00:geographic:00","synset_id":"true.s.07"},{"index":3,"sense_key":"true%3:00:00::","synset_id":"true.a.01"},{"index":4,"sense_key":"true%5:00:00:sincere:00","synset_id":"genuine.s.01"},{"index":5,"sense_key":"true%5:00:00:real:02","synset_id":"true.s.06"},{"index":6,"sense_key":"true%5:00:00:accurate:00","synset_id":"true.s.01"},{"index":7,"sense_key":"true%5:00:00:trustworthy:00","synset_id":"dependable.s.01"},{"index":8,"sense_key":"true%5:00:00:even:01","synset_id":"true.s.10"},{"index":9,"sense_key":"true%5:00:00:typical:00","synset_id":"true.s.03"},{"index":10,"sense_key":"true%5:00:00:legitimate:00","synset_id":"true.s.08"},{"index":11,"sense_key":"true%5:00:00:harmonious:00","synset_id":"on-key.s.01"},{"index":12,"sense_key":"true%5:00:02:faithful:00","synset_id":"true.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"true%3:00:00::","call_ids":["semeval2015.d001.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"true%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":783,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":79},"cost":{"currency":"USD","total_usd":0.006765,"input_uncached_usd":0.003915,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.048644708935171}
{"item_id":"semeval2015.d001.s023.t006","gold_sense_keys":["find%2:32:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":2,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"},{"index":3,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":4,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":5,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":6,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":7,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":8,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":9,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":10,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":11,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":12,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":13,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":14,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":15,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":16,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"find%2:39:02::","call_ids":["semeval2015.d001.s023.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"find%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1058,"cached_input_tokens":0,"output_tokens":389,"reasoning_output_tokens":373},"cost":{"currency":"USD","total_usd":0.01696,"input_uncached_usd":0.00529,"input_cached_usd":0.0,"output_usd":0.01167,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.289344665943645}
{"item_id":"semeval2015.d001.s023.t010","gold_sense_keys":["instance%1:11:00::"],"candidates":[{"index":1,"sense_key":"instance%1:09:00::","synset_id":"example.n.01"},{"index":2,"sense_key":"instance%1:11:00::","synset_id":"case.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"instance%1:11:00::","call_ids":["semeval2015.d001.s023.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"instance%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":363,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":131},"cost":{"currency":"USD","total_usd":0.006195000000000001,"input_uncached_usd":0.0018150000000000002,"input_cached_usd":0.0,"output_usd":0.00438,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.483609791030176}
{"item_id":"semeval2015.d001.s025.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":4,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":5,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":6,"sense_key":"use%2:41:14::","synset_id":"use.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.004945,"input_uncached_usd":0.002605,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8593005000147969}
{"item_id":"semeval2015.d001.s025.t001","gold_sense_keys":["define%2:32:01::"],"candidates":[{"index":1,"sense_key":"define%2:32:00::","synset_id":"define.v.02"},{"index":2,"sense_key":"define%2:42:01::","synset_id":"define.v.03"},{"index":3,"sense_key":"define%2:42:00::","synset_id":"specify.v.03"},{"index":4,"sense_key":"define%2:32:01::","synset_id":"specify.v.02"},{"index":5,"sense_key":"define%2:42:03::","synset_id":"define.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"define%2:32:01::","call_ids":["semeval2015.d001.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"define%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":431,"cached_input_tokens":0,"output_tokens":172,"reasoning_output_tokens":156},"cost":{"currency":"USD","total_usd":0.007315,"input_uncached_usd":0.0021550000000000002,"input_cached_usd":0.0,"output_usd":0.0051600000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5511775409104303}
{"item_id":"semeval2015.d001.s026.t001","gold_sense_keys":["useful%3:00:00::"],"candidates":[{"index":1,"sense_key":"useful%5:00:00:functional:00","synset_id":"utilitarian.s.01"},{"index":2,"sense_key":"useful%3:00:00::","synset_id":"useful.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"useful%3:00:00::","call_ids":["semeval2015.d001.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"useful%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":321,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":110},"cost":{"currency":"USD","total_usd":0.005385,"input_uncached_usd":0.001605,"input_cached_usd":0.0,"output_usd":0.00378,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5761536250356585}
{"item_id":"semeval2015.d001.s026.t002","gold_sense_keys":["work%2:41:02::"],"candidates":[{"index":1,"sense_key":"work%2:39:10::","synset_id":"work.v.18"},{"index":2,"sense_key":"work%2:41:11::","synset_id":"work.v.16"},{"index":3,"sense_key":"work%2:41:01::","synset_id":"work.v.12"},{"index":4,"sense_key":"work%2:41:03::","synset_id":"work.v.03"},{"index":5,"sense_key":"work%2:34:10::","synset_id":"exploit.v.01"},{"index":6,"sense_key":"work%2:35:00::","synset_id":"knead.v.01"},{"index":7,"sense_key":"work%2:29:00::","synset_id":"exercise.v.03"},{"index":8,"sense_key":"work%2:41:12::","synset_id":"work.v.17"},{"index":9,"sense_key":"work%2:36:09::","synset_id":"cultivate.v.02"},{"index":10,"sense_key":"work%2:36:00::","synset_id":"work.v.05"},{"index":11,"sense_key":"work%2:36:12::","synset_id":"shape.v.02"},{"index":12,"sense_key":"work%2:41:02::","synset_id":"work.v.01"},{"index":13,"sense_key":"work%2:30:13::","synset_id":"sour.v.01"},{"index":14,"sense_key":"work%2:41:05::","synset_id":"work.v.09"},{"index":15,"sense_key":"work%2:37:10::","synset_id":"work.v.19"},{"index":16,"sense_key":"work%2:36:01::","synset_id":"bring.v.03"},{"index":17,"sense_key":"work%2:31:13::","synset_id":"solve.v.01"},{"index":18,"sense_key":"work%2:30:10::","synset_id":"work.v.27"},{"index":19,"sense_key":"work%2:41:00::","synset_id":"work.v.02"},{"index":20,"sense_key":"work%2:35:14::","synset_id":"work.v.21"},{"index":21,"sense_key":"work%2:30:14::","synset_id":"ferment.v.03"},{"index":22,"sense_key":"work%2:42:10::","synset_id":"work.v.14"},{"index":23,"sense_key":"work%2:41:10::","synset_id":"influence.v.01"},{"index":24,"sense_key":"work%2:38:00::","synset_id":"make.v.36"},{"index":25,"sense_key":"work%2:38:02::","synset_id":"work.v.10"},{"index":26,"sense_key":"work%2:35:02::","synset_id":"function.v.01"},{"index":27,"sense_key":"work%2:41:04::","synset_id":"work.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"work%2:41:02::","call_ids":["semeval2015.d001.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"work%2:41:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1417,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.022925,"input_uncached_usd":0.007085,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.92686908307951}
{"item_id":"semeval2015.d001.s027.t000","gold_sense_keys":["now%4:02:07::"],"candidates":[{"index":1,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":2,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":3,"sense_key":"now%4:02:06::","synset_id":"now.r.07"},{"index":4,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":5,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":6,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":7,"sense_key":"now%4:02:02::","synset_id":"now.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"now%4:02:07::","call_ids":["semeval2015.d001.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"now%4:02:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":575,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":86},"cost":{"currency":"USD","total_usd":0.0059050000000000005,"input_uncached_usd":0.0028750000000000004,"input_cached_usd":0.0,"output_usd":0.00303,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.743397790938616}
{"item_id":"semeval2015.d001.s027.t001","gold_sense_keys":["ask%2:32:04::"],"candidates":[{"index":1,"sense_key":"ask%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"ask%2:32:05::","synset_id":"ask.v.04"},{"index":3,"sense_key":"ask%2:32:01::","synset_id":"ask.v.02"},{"index":4,"sense_key":"ask%2:32:04::","synset_id":"ask.v.05"},{"index":5,"sense_key":"ask%2:32:09::","synset_id":"ask.v.07"},{"index":6,"sense_key":"ask%2:32:02::","synset_id":"ask.v.03"},{"index":7,"sense_key":"ask%2:32:00::","synset_id":"ask.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"ask%2:32:04::","call_ids":["semeval2015.d001.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"ask%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":106},"cost":{"currency":"USD","total_usd":0.00655,"input_uncached_usd":0.00289,"input_cached_usd":0.0,"output_usd":0.00366,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8966524170245975}
{"item_id":"semeval2015.d001.s027.t002","gold_sense_keys":["user%1:18:00::"],"candidates":[{"index":1,"sense_key":"user%1:18:01::","synset_id":"exploiter.n.01"},{"index":2,"sense_key":"user%1:18:02::","synset_id":"drug_user.n.01"},{"index":3,"sense_key":"user%1:18:00::","synset_id":"user.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"user%1:18:00::","call_ids":["semeval2015.d001.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"user%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":359,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.003325,"input_uncached_usd":0.0017950000000000002,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3448274170514196}
{"item_id":"semeval2015.d001.s028.t000","gold_sense_keys":["easy%3:00:01::"],"candidates":[{"index":1,"sense_key":"easy%5:00:00:gradual:02","synset_id":"easy.s.06"},{"index":2,"sense_key":"easy%3:00:01::","synset_id":"easy.a.01"},{"index":3,"sense_key":"easy%5:00:00:available:00","synset_id":"easy.s.09"},{"index":4,"sense_key":"easy%3:00:02::","synset_id":"easy.a.03"},{"index":5,"sense_key":"easy%5:00:00:comfortable:00","synset_id":"easy.s.07"},{"index":6,"sense_key":"easy%5:00:00:light:08","synset_id":"easy.s.03"},{"index":7,"sense_key":"easy%5:00:00:unchaste:00","synset_id":"easy.s.08"},{"index":8,"sense_key":"easy%5:00:00:abundant:00","synset_id":"easy.s.10"},{"index":9,"sense_key":"easy%5:00:00:pleasing:00","synset_id":"easy.s.02"},{"index":10,"sense_key":"easy%5:00:00:impressionable:00","synset_id":"easy.s.04"},{"index":11,"sense_key":"easy%5:00:00:rich:00","synset_id":"comfortable.s.03"},{"index":12,"sense_key":"easy%5:00:00:unhurried:00","synset_id":"easy.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"easy%3:00:01::","call_ids":["semeval2015.d001.s028.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"easy%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":776,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.0064,"input_uncached_usd":0.00388,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.260786750004627}
{"item_id":"semeval2015.d001.s028.t002","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"},{"index":7,"sense_key":"function%1:04:00::","synset_id":"function.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"function%1:24:00::","call_ids":["semeval2015.d001.s028.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"function%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":124},"cost":{"currency":"USD","total_usd":0.00719,"input_uncached_usd":0.00299,"input_cached_usd":0.0,"output_usd":0.0042,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":135.77039170800708}
{"item_id":"semeval2015.d001.s028.t003","gold_sense_keys":["cos%1:24:00::"],"candidates":[{"index":1,"sense_key":"cobalt%1:27:00::","synset_id":"cobalt.n.01"},{"index":2,"sense_key":"colorado%1:15:00::","synset_id":"colorado.n.01"},{"index":3,"sense_key":"carbon_monoxide%1:27:00::","synset_id":"carbon_monoxide.n.01"},{"index":4,"sense_key":"cos%1:13:00::","synset_id":"cos.n.02"},{"index":5,"sense_key":"conscientious_objector%1:18:00::","synset_id":"conscientious_objector.n.01"},{"index":6,"sense_key":"cos%1:24:00::","synset_id":"cosine.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"cos%1:24:00::","call_ids":["semeval2015.d001.s028.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"cos%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.00422,"input_uncached_usd":0.0025700000000000002,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5813036249019206}
{"item_id":"semeval2015.d001.s028.t004","gold_sense_keys":["sin%1:24:00::"],"candidates":[{"index":1,"sense_key":"sin%1:18:00::","synset_id":"sin.n.04"},{"index":2,"sense_key":"sin%1:04:00::","synset_id":"sin.n.02"},{"index":3,"sense_key":"sin%1:07:00::","synset_id":"sin.n.01"},{"index":4,"sense_key":"sin%1:24:00::","synset_id":"sine.n.01"},{"index":5,"sense_key":"sin%1:10:00::","synset_id":"sin.n.05"},{"index":6,"sense_key":"sin%1:04:01::","synset_id":"sin.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"sin%1:24:00::","call_ids":["semeval2015.d001.s028.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"sin%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00412,"input_uncached_usd":0.00232,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5647253340575844}
{"item_id":"semeval2015.d001.s028.t005","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":2,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%3:00:00::","synset_id":"other.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"other%3:00:00::","call_ids":["semeval2015.d001.s028.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00397,"input_uncached_usd":0.00217,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.003229707945138}
{"item_id":"semeval2015.d001.s028.t008","gold_sense_keys":["sum%1:09:01::"],"candidates":[{"index":1,"sense_key":"sum%1:14:00::","synset_id":"sum.n.03"},{"index":2,"sense_key":"sum%1:09:01::","synset_id":"sum.n.02"},{"index":3,"sense_key":"sum%1:14:01::","synset_id":"union.n.08"},{"index":4,"sense_key":"sum%1:21:00::","synset_id":"sum.n.01"},{"index":5,"sense_key":"sum%1:09:00::","synset_id":"kernel.n.03"},{"index":6,"sense_key":"sum%1:06:00::","synset_id":"sum.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sum%1:09:01::","call_ids":["semeval2015.d001.s028.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sum%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":287,"reasoning_output_tokens":271},"cost":{"currency":"USD","total_usd":0.01123,"input_uncached_usd":0.0026200000000000004,"input_cached_usd":0.0,"output_usd":0.00861,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.249207958928309}
{"item_id":"semeval2015.d001.s028.t009","gold_sense_keys":["product%1:09:00::"],"candidates":[{"index":1,"sense_key":"product%1:06:00::","synset_id":"product.n.02"},{"index":2,"sense_key":"product%1:27:00::","synset_id":"product.n.04"},{"index":3,"sense_key":"product%1:09:00::","synset_id":"product.n.03"},{"index":4,"sense_key":"product%1:06:01::","synset_id":"merchandise.n.01"},{"index":5,"sense_key":"product%1:14:00::","synset_id":"intersection.n.04"},{"index":6,"sense_key":"product%1:19:00::","synset_id":"product.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"product%1:09:00::","call_ids":["semeval2015.d001.s028.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"product%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":540,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.00537,"input_uncached_usd":0.0027,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1707125840475783}
{"item_id":"semeval2015.d001.s030.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":3,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":4,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":5,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":6,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.004360000000000001,"input_uncached_usd":0.0026200000000000004,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5765663330676034}
{"item_id":"semeval2015.d001.s030.t001","gold_sense_keys":["plus%1:04:00::"],"candidates":[{"index":1,"sense_key":"plus%1:07:00::","synset_id":"asset.n.01"},{"index":2,"sense_key":"plus%1:04:00::","synset_id":"summation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"plus%1:04:00::","call_ids":["semeval2015.d001.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"plus%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":323,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.003925,"input_uncached_usd":0.0016150000000000001,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.028137124958448}
{"item_id":"semeval2015.d001.s030.t002","gold_sense_keys":["times%1:04:00::"],"candidates":[{"index":1,"sense_key":"meter%1:07:00::","synset_id":"meter.n.04"},{"index":2,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":3,"sense_key":"clock_time%1:28:00::","synset_id":"clock_time.n.01"},{"index":4,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":5,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":6,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":7,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":8,"sense_key":"times%1:04:00::","synset_id":"multiplication.n.03"},{"index":9,"sense_key":"times%1:28:01::","synset_id":"times.n.01"},{"index":10,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":11,"sense_key":"fourth_dimension%1:28:00::","synset_id":"fourth_dimension.n.01"},{"index":12,"sense_key":"prison_term%1:28:00::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"times%1:04:00::","call_ids":["semeval2015.d001.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"times%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.005785,"input_uncached_usd":0.0038650000000000004,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9054313329979777}
{"item_id":"semeval2015.d001.s030.t003","gold_sense_keys":["have%2:40:00::","have%2:42:00::"],"candidates":[{"index":1,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":2,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":3,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":4,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":5,"sense_key":"have%2:35:00::","synset_id":"take.v.35"},{"index":6,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":9,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":13,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":14,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":15,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":16,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":17,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":18,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":19,"sense_key":"have%2:40:04::","synset_id":"own.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"have%2:42:00::","call_ids":["semeval2015.d001.s030.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"have%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1078,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.021200000000000004,"input_uncached_usd":0.005390000000000001,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.728622292052023}
{"item_id":"semeval2015.d001.s030.t004","gold_sense_keys":["operator%1:24:00::"],"candidates":[{"index":1,"sense_key":"operator%1:24:00::","synset_id":"operator.n.01"},{"index":2,"sense_key":"operator%1:18:02::","synset_id":"hustler.n.02"},{"index":3,"sense_key":"operator%1:18:01::","synset_id":"operator.n.03"},{"index":4,"sense_key":"operator%1:18:00::","synset_id":"operator.n.02"},{"index":5,"sense_key":"operator%1:18:03::","synset_id":"operator.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"operator%1:24:00::","call_ids":["semeval2015.d001.s030.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"operator%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":411,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.003645,"input_uncached_usd":0.002055,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5609241250203922}
{"item_id":"semeval2015.d001.s031.t000","gold_sense_keys":["useful%3:00:00::"],"candidates":[{"index":1,"sense_key":"useful%5:00:00:functional:00","synset_id":"utilitarian.s.01"},{"index":2,"sense_key":"useful%3:00:00::","synset_id":"useful.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"useful%3:00:00::","call_ids":["semeval2015.d001.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"useful%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":323,"cached_input_tokens":0,"output_tokens":160,"reasoning_output_tokens":145},"cost":{"currency":"USD","total_usd":0.006415,"input_uncached_usd":0.0016150000000000001,"input_cached_usd":0.0,"output_usd":0.0048000000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.706740584108047}
{"item_id":"semeval2015.d001.s032.t000","gold_sense_keys":["there%4:02:00::"],"candidates":[{"index":1,"sense_key":"there%4:02:00::","synset_id":"there.r.01"},{"index":2,"sense_key":"there%4:02:01::","synset_id":"there.r.03"},{"index":3,"sense_key":"there%4:02:02::","synset_id":"there.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"there%4:02:00::","call_ids":["semeval2015.d001.s032.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"there%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":379,"cached_input_tokens":0,"output_tokens":170,"reasoning_output_tokens":155},"cost":{"currency":"USD","total_usd":0.006995,"input_uncached_usd":0.0018950000000000002,"input_cached_usd":0.0,"output_usd":0.0051,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.088183332933113}
{"item_id":"semeval2015.d001.s032.t001","gold_sense_keys":["have%2:42:12::"],"candidates":[{"index":1,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":2,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":3,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":4,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":5,"sense_key":"have%2:35:00::","synset_id":"take.v.35"},{"index":6,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":7,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":8,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":9,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":10,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":11,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":12,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":13,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":14,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":15,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":16,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":17,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":18,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":19,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"have%2:40:00::","call_ids":["semeval2015.d001.s032.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"have%2:40:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1096,"cached_input_tokens":0,"output_tokens":953,"reasoning_output_tokens":932},"cost":{"currency":"USD","total_usd":0.03407,"input_uncached_usd":0.0054800000000000005,"input_cached_usd":0.0,"output_usd":0.02859,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":21.17396187502891}
{"item_id":"semeval2015.d001.s032.t002","gold_sense_keys":["list%1:10:00::"],"candidates":[{"index":1,"sense_key":"list%1:10:00::","synset_id":"list.n.01"},{"index":2,"sense_key":"list%1:07:00::","synset_id":"tilt.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"list%1:10:00::","call_ids":["semeval2015.d001.s032.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"list%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":351,"cached_input_tokens":0,"output_tokens":40,"reasoning_output_tokens":25},"cost":{"currency":"USD","total_usd":0.002955,"input_uncached_usd":0.001755,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4356242089997977}
{"item_id":"semeval2015.d001.s032.t003","gold_sense_keys":["variable%1:10:00::"],"candidates":[{"index":1,"sense_key":"variable%1:17:00::","synset_id":"variable.n.01"},{"index":2,"sense_key":"variable%1:17:01::","synset_id":"variable_star.n.01"},{"index":3,"sense_key":"variable%1:10:00::","synset_id":"variable.n.04"},{"index":4,"sense_key":"variable%1:09:00::","synset_id":"variable.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"variable%1:10:00::","call_ids":["semeval2015.d001.s032.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"variable%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":397,"cached_input_tokens":0,"output_tokens":235,"reasoning_output_tokens":220},"cost":{"currency":"USD","total_usd":0.009035,"input_uncached_usd":0.0019850000000000002,"input_cached_usd":0.0,"output_usd":0.00705,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.3124972499208525}
{"item_id":"semeval2015.d001.s033.t000","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":2,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":3,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":6,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":7,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"},{"index":8,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":9,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":10,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":11,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":12,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":13,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":14,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":15,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":16,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":17,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":18,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":19,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":20,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":21,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":22,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":23,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":24,"sense_key":"see%2:39:05::","synset_id":"see.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":23,"chosen_sense_key":"see%2:39:00::","call_ids":["semeval2015.d001.s033.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":23,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1410,"cached_input_tokens":0,"output_tokens":194,"reasoning_output_tokens":178},"cost":{"currency":"USD","total_usd":0.012870000000000001,"input_uncached_usd":0.007050000000000001,"input_cached_usd":0.0,"output_usd":0.0058200000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.69408825004939}
{"item_id":"semeval2015.d001.s033.t001","gold_sense_keys":["let%2:41:00::"],"candidates":[{"index":1,"sense_key":"let%2:40:00::","synset_id":"lease.v.03"},{"index":2,"sense_key":"let%2:32:02::","synset_id":"permit.v.01"},{"index":3,"sense_key":"let%2:30:00::","synset_id":"let.v.05"},{"index":4,"sense_key":"let%2:30:01::","synset_id":"get.v.03"},{"index":5,"sense_key":"let%2:41:00::","synset_id":"let.v.01"},{"index":6,"sense_key":"let%2:32:00::","synset_id":"let.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"let%2:41:00::","call_ids":["semeval2015.d001.s033.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"let%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":539,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":108},"cost":{"currency":"USD","total_usd":0.0064150000000000006,"input_uncached_usd":0.0026950000000000003,"input_cached_usd":0.0,"output_usd":0.00372,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.359812375041656}
{"item_id":"semeval2015.d001.s033.t002","gold_sense_keys":["change%2:30:01::"],"candidates":[{"index":1,"sense_key":"change%2:30:00::","synset_id":"change.v.02"},{"index":2,"sense_key":"change%2:30:03::","synset_id":"change.v.05"},{"index":3,"sense_key":"change%2:30:05::","synset_id":"switch.v.03"},{"index":4,"sense_key":"change%2:30:08::","synset_id":"deepen.v.04"},{"index":5,"sense_key":"change%2:30:04::","synset_id":"change.v.06"},{"index":6,"sense_key":"change%2:30:06::","synset_id":"change.v.10"},{"index":7,"sense_key":"change%2:40:00::","synset_id":"exchange.v.01"},{"index":8,"sense_key":"change%2:30:02::","synset_id":"change.v.03"},{"index":9,"sense_key":"change%2:38:00::","synset_id":"transfer.v.06"},{"index":10,"sense_key":"change%2:30:01::","synset_id":"change.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"change%2:30:01::","call_ids":["semeval2015.d001.s033.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"change%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.0055650000000000005,"input_uncached_usd":0.0038250000000000003,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.454832165967673}
{"item_id":"semeval2015.d001.s033.t003","gold_sense_keys":["value%1:09:00::"],"candidates":[{"index":1,"sense_key":"value%1:28:00::","synset_id":"value.n.05"},{"index":2,"sense_key":"value%1:07:00::","synset_id":"value.n.02"},{"index":3,"sense_key":"value%1:21:00::","synset_id":"value.n.03"},{"index":4,"sense_key":"value%1:09:01::","synset_id":"value.n.06"},{"index":5,"sense_key":"value%1:09:00::","synset_id":"value.n.01"},{"index":6,"sense_key":"value%1:07:01::","synset_id":"value.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"value%1:09:00::","call_ids":["semeval2015.d001.s033.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"value%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004395,"input_uncached_usd":0.002595,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2637578749563545}
{"item_id":"semeval2015.d001.s033.t004","gold_sense_keys":["just%4:02:00::"],"candidates":[{"index":1,"sense_key":"just%4:02:03::","synset_id":"just.r.04"},{"index":2,"sense_key":"just%4:02:04::","synset_id":"barely.r.01"},{"index":3,"sense_key":"just%4:02:00::","synset_id":"merely.r.01"},{"index":4,"sense_key":"just%4:02:06::","synset_id":"just.r.06"},{"index":5,"sense_key":"just%4:02:05::","synset_id":"precisely.r.01"},{"index":6,"sense_key":"just%4:02:01::","synset_id":"just.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"just%4:02:00::","call_ids":["semeval2015.d001.s033.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"just%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":554,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":73},"cost":{"currency":"USD","total_usd":0.00541,"input_uncached_usd":0.0027700000000000003,"input_cached_usd":0.0,"output_usd":0.00264,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0556228749919683}
{"item_id":"semeval2015.d001.s033.t005","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":3,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":4,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":5,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":8,"sense_key":"way%1:21:00::","synset_id":"way.n.12"},{"index":9,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":10,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":11,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":12,"sense_key":"way%1:09:01::","synset_id":"way.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"way%1:04:01::","call_ids":["semeval2015.d001.s033.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":99},"cost":{"currency":"USD","total_usd":0.007135000000000001,"input_uncached_usd":0.0036850000000000003,"input_cached_usd":0.0,"output_usd":0.00345,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.053840082953684}
{"item_id":"semeval2015.d001.s033.t007","gold_sense_keys":["log%1:10:02::"],"candidates":[{"index":1,"sense_key":"log%1:27:01::","synset_id":"log.n.01"},{"index":2,"sense_key":"log%1:10:00::","synset_id":"log.n.04"},{"index":3,"sense_key":"log%1:06:01::","synset_id":"log.n.05"},{"index":4,"sense_key":"log%1:10:02::","synset_id":"log.n.03"},{"index":5,"sense_key":"log%1:10:01::","synset_id":"logarithm.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"log%1:10:02::","call_ids":["semeval2015.d001.s033.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"log%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01805,"input_uncached_usd":0.00221,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.394970125053078}
{"item_id":"semeval2015.d001.s034.t000","gold_sense_keys":["time%1:11:00::"],"candidates":[{"index":1,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":2,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":3,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":4,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":5,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":6,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":7,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":8,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":9,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"},{"index":10,"sense_key":"time%1:28:00::","synset_id":"time.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"time%1:11:00::","call_ids":["semeval2015.d001.s034.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"time%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.005035,"input_uncached_usd":0.0032350000000000005,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4731355829862878}
{"item_id":"semeval2015.d001.s034.t002","gold_sense_keys":["expression%1:10:03::"],"candidates":[{"index":1,"sense_key":"expression%1:07:00::","synset_id":"expression.n.01"},{"index":2,"sense_key":"expression%1:10:05::","synset_id":"construction.n.02"},{"index":3,"sense_key":"expression%1:10:00::","synset_id":"saying.n.01"},{"index":4,"sense_key":"expression%1:10:03::","synset_id":"formula.n.01"},{"index":5,"sense_key":"expression%1:10:02::","synset_id":"expression.n.02"},{"index":6,"sense_key":"expression%1:22:00::","synset_id":"expression.n.07"},{"index":7,"sense_key":"expression%1:04:00::","synset_id":"expression.n.09"},{"index":8,"sense_key":"expression%1:10:04::","synset_id":"expression.n.03"},{"index":9,"sense_key":"expression%1:10:01::","synset_id":"formulation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"expression%1:10:03::","call_ids":["semeval2015.d001.s034.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"expression%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.00523,"input_uncached_usd":0.0031300000000000004,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":100.59456008393317}
{"item_id":"semeval2015.d001.s034.t003","gold_sense_keys":["variable%1:10:00::"],"candidates":[{"index":1,"sense_key":"variable%1:17:00::","synset_id":"variable.n.01"},{"index":2,"sense_key":"variable%1:09:00::","synset_id":"variable.n.02"},{"index":3,"sense_key":"variable%1:10:00::","synset_id":"variable.n.04"},{"index":4,"sense_key":"variable%1:17:01::","synset_id":"variable_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"variable%1:10:00::","call_ids":["semeval2015.d001.s034.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"variable%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":380,"cached_input_tokens":0,"output_tokens":304,"reasoning_output_tokens":288},"cost":{"currency":"USD","total_usd":0.01102,"input_uncached_usd":0.0019000000000000002,"input_cached_usd":0.0,"output_usd":0.00912,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.222397083998658}
{"item_id":"semeval2015.d001.s034.t004","gold_sense_keys":["value%1:09:00::"],"candidates":[{"index":1,"sense_key":"value%1:28:00::","synset_id":"value.n.05"},{"index":2,"sense_key":"value%1:09:00::","synset_id":"value.n.01"},{"index":3,"sense_key":"value%1:21:00::","synset_id":"value.n.03"},{"index":4,"sense_key":"value%1:07:00::","synset_id":"value.n.02"},{"index":5,"sense_key":"value%1:09:01::","synset_id":"value.n.06"},{"index":6,"sense_key":"value%1:07:01::","synset_id":"value.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"value%1:09:00::","call_ids":["semeval2015.d001.s034.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"value%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004485,"input_uncached_usd":0.002475,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8256403749110177}
{"item_id":"semeval2015.d001.s034.t005","gold_sense_keys":["change%2:30:01::"],"candidates":[{"index":1,"sense_key":"change%2:30:05::","synset_id":"switch.v.03"},{"index":2,"sense_key":"change%2:30:01::","synset_id":"change.v.01"},{"index":3,"sense_key":"change%2:30:08::","synset_id":"deepen.v.04"},{"index":4,"sense_key":"change%2:30:03::","synset_id":"change.v.05"},{"index":5,"sense_key":"change%2:38:00::","synset_id":"transfer.v.06"},{"index":6,"sense_key":"change%2:30:02::","synset_id":"change.v.03"},{"index":7,"sense_key":"change%2:30:00::","synset_id":"change.v.02"},{"index":8,"sense_key":"change%2:40:00::","synset_id":"exchange.v.01"},{"index":9,"sense_key":"change%2:30:04::","synset_id":"change.v.06"},{"index":10,"sense_key":"change%2:30:06::","synset_id":"change.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"change%2:30:01::","call_ids":["semeval2015.d001.s034.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"change%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":741,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019545,"input_uncached_usd":0.0037050000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.006012417026795}
{"item_id":"semeval2015.d001.s034.t006","gold_sense_keys":["last%3:00:00::","last%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"last%5:00:00:closing:00","synset_id":"concluding.s.01"},{"index":2,"sense_key":"last%5:00:00:worst:00","synset_id":"last.s.08"},{"index":3,"sense_key":"last%5:00:00:unlikely:00","synset_id":"last.s.03"},{"index":4,"sense_key":"last%5:00:00:high:02","synset_id":"last.s.06"},{"index":5,"sense_key":"last%5:00:00:past:00","synset_id":"last.s.01"},{"index":6,"sense_key":"last%3:00:00::","synset_id":"last.a.02"},{"index":7,"sense_key":"last%5:00:00:ultimate:00","synset_id":"final.s.02"},{"index":8,"sense_key":"last%5:00:00:dying:00","synset_id":"last.s.04"},{"index":9,"sense_key":"last%5:00:00:unalterable:00","synset_id":"final.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"last%5:00:00:past:00","call_ids":["semeval2015.d001.s034.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"last%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":655,"cached_input_tokens":0,"output_tokens":804,"reasoning_output_tokens":783},"cost":{"currency":"USD","total_usd":0.027395,"input_uncached_usd":0.003275,"input_cached_usd":0.0,"output_usd":0.02412,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":16.176024375017732}
{"item_id":"semeval2015.d001.s034.t007","gold_sense_keys":["result%1:10:00::","result%1:11:00::"],"candidates":[{"index":1,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":2,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"},{"index":3,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":4,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"result%1:10:00::","call_ids":["semeval2015.d001.s034.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"result%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018095,"input_uncached_usd":0.002255,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.664914000080898}
{"item_id":"semeval2015.d001.s035.t000","gold_sense_keys":["new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":6,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":7,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":8,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":9,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":10,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":11,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"new%3:00:00::","call_ids":["semeval2015.d001.s035.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"new%3:00:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":161,"reasoning_output_tokens":145},"cost":{"currency":"USD","total_usd":0.008435,"input_uncached_usd":0.003605,"input_cached_usd":0.0,"output_usd":0.00483,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.410694458987564}
{"item_id":"semeval2015.d001.s035.t001","gold_sense_keys":["window%1:06:03::"],"candidates":[{"index":1,"sense_key":"window%1:06:00::","synset_id":"window.n.01"},{"index":2,"sense_key":"window%1:17:00::","synset_id":"window.n.04"},{"index":3,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":4,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":5,"sense_key":"window%1:06:03::","synset_id":"window.n.08"},{"index":6,"sense_key":"window%1:28:00::","synset_id":"window.n.05"},{"index":7,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":8,"sense_key":"window%1:06:01::","synset_id":"window.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"window%1:06:03::","call_ids":["semeval2015.d001.s035.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"window%1:06:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":566,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.00541,"input_uncached_usd":0.00283,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.193777584005147}
{"item_id":"semeval2015.d001.s036.t000","gold_sense_keys":["log%1:10:02::"],"candidates":[{"index":1,"sense_key":"log%1:06:01::","synset_id":"log.n.05"},{"index":2,"sense_key":"log%1:10:01::","synset_id":"logarithm.n.01"},{"index":3,"sense_key":"log%1:10:02::","synset_id":"log.n.03"},{"index":4,"sense_key":"log%1:27:01::","synset_id":"log.n.01"},{"index":5,"sense_key":"log%1:10:00::","synset_id":"log.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"log%1:10:02::","call_ids":["semeval2015.d001.s036.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"log%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":190,"reasoning_output_tokens":175},"cost":{"currency":"USD","total_usd":0.00772,"input_uncached_usd":0.00202,"input_cached_usd":0.0,"output_usd":0.0057,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.2981706249993294}
{"item_id":"semeval2015.d001.s037.t000","gold_sense_keys":["load%2:40:12::"],"candidates":[{"index":1,"sense_key":"load%2:35:02::","synset_id":"load.v.02"},{"index":2,"sense_key":"load%2:35:01::","synset_id":"load.v.01"},{"index":3,"sense_key":"load%2:40:12::","synset_id":"load.v.03"},{"index":4,"sense_key":"load%2:30:12::","synset_id":"load.v.05"},{"index":5,"sense_key":"load%2:35:10::","synset_id":"load.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"load%2:40:12::","call_ids":["semeval2015.d001.s037.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"load%2:40:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.005275,"input_uncached_usd":0.0021850000000000003,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4911747919395566}
{"item_id":"semeval2015.d001.s038.t000","gold_sense_keys":["execute%2:41:04::"],"candidates":[{"index":1,"sense_key":"execute%2:41:03::","synset_id":"execute.v.04"},{"index":2,"sense_key":"execute%2:41:00::","synset_id":"execute.v.01"},{"index":3,"sense_key":"execute%2:41:04::","synset_id":"run.v.19"},{"index":4,"sense_key":"execute%2:36:01::","synset_id":"perform.v.01"},{"index":5,"sense_key":"execute%2:41:01::","synset_id":"execute.v.02"},{"index":6,"sense_key":"execute%2:32:00::","synset_id":"execute.v.07"},{"index":7,"sense_key":"execute%2:36:00::","synset_id":"carry_through.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"execute%2:41:04::","call_ids":["semeval2015.d001.s038.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"execute%2:41:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":93},"cost":{"currency":"USD","total_usd":0.005900000000000001,"input_uncached_usd":0.0026300000000000004,"input_cached_usd":0.0,"output_usd":0.00327,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2511887090513483}
{"item_id":"semeval2015.d001.s038.t001","gold_sense_keys":["instruction%1:10:02::"],"candidates":[{"index":1,"sense_key":"instruction%1:04:01::","synset_id":"education.n.01"},{"index":2,"sense_key":"instruction%1:10:04::","synset_id":"direction.n.06"},{"index":3,"sense_key":"instruction%1:10:02::","synset_id":"instruction.n.04"},{"index":4,"sense_key":"instruction%1:04:00::","synset_id":"teaching.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"instruction%1:10:02::","call_ids":["semeval2015.d001.s038.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"instruction%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":421,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.004505,"input_uncached_usd":0.002105,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6091796250548214}
{"item_id":"semeval2015.d001.s038.t002","gold_sense_keys":["file%1:10:00::"],"candidates":[{"index":1,"sense_key":"file%1:06:01::","synset_id":"file.n.03"},{"index":2,"sense_key":"file%1:10:00::","synset_id":"file.n.01"},{"index":3,"sense_key":"file%1:06:00::","synset_id":"file.n.04"},{"index":4,"sense_key":"file%1:14:00::","synset_id":"file.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"file%1:10:00::","call_ids":["semeval2015.d001.s038.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"file%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":367,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.003755,"input_uncached_usd":0.001835,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9513459169538692}
{"item_id":"semeval2015.d001.s039.t000","gold_sense_keys":["nice%3:00:00::"],"candidates":[{"index":1,"sense_key":"nice%5:00:00:polite:00","synset_id":"courteous.s.01"},{"index":2,"sense_key":"nice%5:00:00:fastidious:00","synset_id":"dainty.s.04"},{"index":3,"sense_key":"nice%5:00:02:precise:00","synset_id":"nice.s.02"},{"index":4,"sense_key":"nice%5:00:00:respectable:00","synset_id":"decent.s.01"},{"index":5,"sense_key":"nice%3:00:00::","synset_id":"nice.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"nice%3:00:00::","call_ids":["semeval2015.d001.s039.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"nice%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":79},"cost":{"currency":"USD","total_usd":0.005195,"input_uncached_usd":0.0023450000000000003,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8068652090150863}
{"item_id":"semeval2015.d001.s039.t001","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":2,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":3,"sense_key":"want%2:42:01::","synset_id":"want.v.05"},{"index":4,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":5,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"want%2:37:00::","call_ids":["semeval2015.d001.s039.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.0040750000000000005,"input_uncached_usd":0.0020350000000000004,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6744274999946356}
{"item_id":"semeval2015.d001.s039.t002","gold_sense_keys":["define%2:32:01::"],"candidates":[{"index":1,"sense_key":"define%2:42:03::","synset_id":"define.v.04"},{"index":2,"sense_key":"define%2:32:01::","synset_id":"specify.v.02"},{"index":3,"sense_key":"define%2:42:01::","synset_id":"define.v.03"},{"index":4,"sense_key":"define%2:32:00::","synset_id":"define.v.02"},{"index":5,"sense_key":"define%2:42:00::","synset_id":"specify.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"define%2:32:01::","call_ids":["semeval2015.d001.s039.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"define%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":148},"cost":{"currency":"USD","total_usd":0.0069,"input_uncached_usd":0.00201,"input_cached_usd":0.0,"output_usd":0.00489,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.432630750001408}
{"item_id":"semeval2015.d001.s039.t003","gold_sense_keys":["library%1:14:01::"],"candidates":[{"index":1,"sense_key":"library%1:14:01::","synset_id":"library.n.04"},{"index":2,"sense_key":"library%1:06:02::","synset_id":"library.n.05"},{"index":3,"sense_key":"library%1:06:01::","synset_id":"library.n.01"},{"index":4,"sense_key":"library%1:14:00::","synset_id":"library.n.02"},{"index":5,"sense_key":"library%1:06:00::","synset_id":"library.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"library%1:14:01::","call_ids":["semeval2015.d001.s039.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"library%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":379,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.0042650000000000006,"input_uncached_usd":0.0018950000000000002,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7532910000300035}
{"item_id":"semeval2015.d001.s039.t004","gold_sense_keys":["resume%2:30:00::"],"candidates":[{"index":1,"sense_key":"resume%2:41:00::","synset_id":"resume.v.03"},{"index":2,"sense_key":"resume%2:32:00::","synset_id":"sum_up.v.01"},{"index":3,"sense_key":"resume%2:30:01::","synset_id":"resume.v.02"},{"index":4,"sense_key":"resume%2:30:00::","synset_id":"resume.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"resume%2:30:00::","call_ids":["semeval2015.d001.s039.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"resume%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":360,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":93},"cost":{"currency":"USD","total_usd":0.00507,"input_uncached_usd":0.0018000000000000002,"input_cached_usd":0.0,"output_usd":0.00327,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.718847915995866}
{"item_id":"semeval2015.d001.s039.t005","gold_sense_keys":["previous%5:00:00:preceding:00"],"candidates":[{"index":1,"sense_key":"previous%5:00:00:past:00","synset_id":"former.s.02"},{"index":2,"sense_key":"previous%5:00:00:preceding:00","synset_id":"previous.s.01"},{"index":3,"sense_key":"previous%5:00:00:early:00","synset_id":"previous.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"previous%5:00:00:preceding:00","call_ids":["semeval2015.d001.s039.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"previous%5:00:00:preceding:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":350,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.00394,"input_uncached_usd":0.00175,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8756455420516431}
{"item_id":"semeval2015.d001.s039.t006","gold_sense_keys":["work%1:04:00::"],"candidates":[{"index":1,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":2,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":3,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"},{"index":4,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":5,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":6,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":7,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"work%1:04:00::","call_ids":["semeval2015.d001.s039.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"work%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":146},"cost":{"currency":"USD","total_usd":0.007710000000000001,"input_uncached_usd":0.00285,"input_cached_usd":0.0,"output_usd":0.00486,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4068368340376765}
{"item_id":"semeval2015.d001.s040.t000","gold_sense_keys":["save%2:32:04::"],"candidates":[{"index":1,"sense_key":"save%2:40:00::","synset_id":"save.v.02"},{"index":2,"sense_key":"save%2:30:00::","synset_id":"save.v.06"},{"index":3,"sense_key":"save%2:41:00::","synset_id":"save.v.03"},{"index":4,"sense_key":"save%2:40:03::","synset_id":"keep_open.v.01"},{"index":5,"sense_key":"save%2:40:02::","synset_id":"save.v.04"},{"index":6,"sense_key":"save%2:40:01::","synset_id":"save.v.05"},{"index":7,"sense_key":"save%2:32:04::","synset_id":"write.v.08"},{"index":8,"sense_key":"save%2:40:04::","synset_id":"save.v.09"},{"index":9,"sense_key":"save%2:41:05::","synset_id":"spare.v.01"},{"index":10,"sense_key":"save%2:41:01::","synset_id":"salvage.v.01"},{"index":11,"sense_key":"save%2:41:02::","synset_id":"deliver.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"save%2:32:04::","call_ids":["semeval2015.d001.s040.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"save%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.005615,"input_uncached_usd":0.003125,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9430312080075964}
{"item_id":"semeval2015.d001.s041.t000","gold_sense_keys":["save%2:32:04::"],"candidates":[{"index":1,"sense_key":"save%2:41:02::","synset_id":"deliver.v.08"},{"index":2,"sense_key":"save%2:40:04::","synset_id":"save.v.09"},{"index":3,"sense_key":"save%2:41:00::","synset_id":"save.v.03"},{"index":4,"sense_key":"save%2:40:00::","synset_id":"save.v.02"},{"index":5,"sense_key":"save%2:40:03::","synset_id":"keep_open.v.01"},{"index":6,"sense_key":"save%2:41:01::","synset_id":"salvage.v.01"},{"index":7,"sense_key":"save%2:41:05::","synset_id":"spare.v.01"},{"index":8,"sense_key":"save%2:30:00::","synset_id":"save.v.06"},{"index":9,"sense_key":"save%2:32:04::","synset_id":"write.v.08"},{"index":10,"sense_key":"save%2:40:01::","synset_id":"save.v.05"},{"index":11,"sense_key":"save%2:40:02::","synset_id":"save.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"save%2:32:04::","call_ids":["semeval2015.d001.s041.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"save%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.005685000000000001,"input_uncached_usd":0.0031950000000000004,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6018220420228317}
{"item_id":"semeval2015.d001.s041.t001","gold_sense_keys":["instruction%1:10:02::"],"candidates":[{"index":1,"sense_key":"instruction%1:04:00::","synset_id":"teaching.n.01"},{"index":2,"sense_key":"instruction%1:10:02::","synset_id":"instruction.n.04"},{"index":3,"sense_key":"instruction%1:04:01::","synset_id":"education.n.01"},{"index":4,"sense_key":"instruction%1:10:04::","synset_id":"direction.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"instruction%1:10:02::","call_ids":["semeval2015.d001.s041.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"instruction%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.005135000000000001,"input_uncached_usd":0.0020450000000000004,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.274755750084296}
{"item_id":"semeval2015.d001.s041.t002","gold_sense_keys":["type%2:32:00::"],"candidates":[{"index":1,"sense_key":"type%2:31:00::","synset_id":"type.v.02"},{"index":2,"sense_key":"type%2:32:00::","synset_id":"type.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"type%2:32:00::","call_ids":["semeval2015.d001.s041.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"type%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":295,"cached_input_tokens":0,"output_tokens":42,"reasoning_output_tokens":27},"cost":{"currency":"USD","total_usd":0.0027350000000000005,"input_uncached_usd":0.0014750000000000002,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.634819082915783}
{"item_id":"semeval2015.d001.s041.t003","gold_sense_keys":["begin%2:42:00::","begin%2:42:04::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":3,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":4,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":5,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"},{"index":6,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":7,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":8,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":9,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":10,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"begin%2:42:04::","call_ids":["semeval2015.d001.s041.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"begin%2:42:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":727,"cached_input_tokens":0,"output_tokens":256,"reasoning_output_tokens":240},"cost":{"currency":"USD","total_usd":0.011315,"input_uncached_usd":0.003635,"input_cached_usd":0.0,"output_usd":0.00768,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.353149415925145}
{"item_id":"semeval2015.d001.s041.t004","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"},{"index":2,"sense_key":"able%3:00:00::","synset_id":"able.a.01"},{"index":3,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"},{"index":4,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"able%3:00:00::","call_ids":["semeval2015.d001.s041.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"able%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":405,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":133},"cost":{"currency":"USD","total_usd":0.006495000000000001,"input_uncached_usd":0.0020250000000000003,"input_cached_usd":0.0,"output_usd":0.00447,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.464870832976885}
{"item_id":"semeval2015.d001.s042.t000","gold_sense_keys":["generate%2:36:00::"],"candidates":[{"index":1,"sense_key":"generate%2:29:00::","synset_id":"beget.v.01"},{"index":2,"sense_key":"generate%2:36:00::","synset_id":"generate.v.01"},{"index":3,"sense_key":"generate%2:36:01::","synset_id":"generate.v.03"},{"index":4,"sense_key":"generate%2:36:02::","synset_id":"render.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"generate%2:36:00::","call_ids":["semeval2015.d001.s042.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"generate%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.004475,"input_uncached_usd":0.002075,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.702071125037037}
{"item_id":"semeval2015.d001.s042.t002","gold_sense_keys":["easy%3:00:01::"],"candidates":[{"index":1,"sense_key":"easy%5:00:00:rich:00","synset_id":"comfortable.s.03"},{"index":2,"sense_key":"easy%5:00:00:comfortable:00","synset_id":"easy.s.07"},{"index":3,"sense_key":"easy%5:00:00:abundant:00","synset_id":"easy.s.10"},{"index":4,"sense_key":"easy%5:00:00:gradual:02","synset_id":"easy.s.06"},{"index":5,"sense_key":"easy%3:00:01::","synset_id":"easy.a.01"},{"index":6,"sense_key":"easy%5:00:00:unhurried:00","synset_id":"easy.s.01"},{"index":7,"sense_key":"easy%5:00:00:pleasing:00","synset_id":"easy.s.02"},{"index":8,"sense_key":"easy%5:00:00:unchaste:00","synset_id":"easy.s.08"},{"index":9,"sense_key":"easy%5:00:00:impressionable:00","synset_id":"easy.s.04"},{"index":10,"sense_key":"easy%3:00:02::","synset_id":"easy.a.03"},{"index":11,"sense_key":"easy%5:00:00:light:08","synset_id":"easy.s.03"},{"index":12,"sense_key":"easy%5:00:00:available:00","synset_id":"easy.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"easy%3:00:01::","call_ids":["semeval2015.d001.s042.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"easy%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.006015,"input_uncached_usd":0.0036750000000000003,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0751867089420557}
{"item_id":"semeval2015.d001.s042.t003","gold_sense_keys":["fix%2:30:01::"],"candidates":[{"index":1,"sense_key":"fix%2:32:03::","synset_id":"specify.v.02"},{"index":2,"sense_key":"fix%2:35:01::","synset_id":"situate.v.02"},{"index":3,"sense_key":"fix%2:30:01::","synset_id":"repair.v.01"},{"index":4,"sense_key":"fix%2:41:04::","synset_id":"fix.v.10"},{"index":5,"sense_key":"fix%2:35:00::","synset_id":"fasten.v.01"},{"index":6,"sense_key":"fix%2:33:00::","synset_id":"pay_back.v.02"},{"index":7,"sense_key":"fix%2:30:03::","synset_id":"fix.v.12"},{"index":8,"sense_key":"fix%2:29:00::","synset_id":"sterilize.v.02"},{"index":9,"sense_key":"fix%2:36:01::","synset_id":"fix.v.06"},{"index":10,"sense_key":"fix%2:30:02::","synset_id":"fix.v.07"},{"index":11,"sense_key":"fix%2:36:00::","synset_id":"cook.v.02"},{"index":12,"sense_key":"fix%2:30:00::","synset_id":"fixate.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fix%2:30:01::","call_ids":["semeval2015.d001.s042.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fix%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":778,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.006710000000000001,"input_uncached_usd":0.0038900000000000002,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8396487080026418}
{"item_id":"semeval2015.d001.s042.t004","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":3,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":4,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:34:02::","synset_id":"use.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s042.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.00496,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6827741250162944}
{"item_id":"semeval2015.d001.s042.t005","gold_sense_keys":["text_editor%1:10:00::"],"candidates":[{"index":1,"sense_key":"text_editor%1:10:00::","synset_id":"text_editor.n.02"},{"index":2,"sense_key":"text_editor%1:18:00::","synset_id":"copy_editor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"text_editor%1:10:00::","call_ids":["semeval2015.d001.s042.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"text_editor%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":285,"cached_input_tokens":0,"output_tokens":32,"reasoning_output_tokens":17},"cost":{"currency":"USD","total_usd":0.002385,"input_uncached_usd":0.001425,"input_cached_usd":0.0,"output_usd":0.00096,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2866682920139283}
{"item_id":"semeval2015.d001.s043.t000","gold_sense_keys":["save%2:32:04::"],"candidates":[{"index":1,"sense_key":"save%2:40:02::","synset_id":"save.v.04"},{"index":2,"sense_key":"save%2:41:01::","synset_id":"salvage.v.01"},{"index":3,"sense_key":"save%2:32:04::","synset_id":"write.v.08"},{"index":4,"sense_key":"save%2:40:00::","synset_id":"save.v.02"},{"index":5,"sense_key":"save%2:40:03::","synset_id":"keep_open.v.01"},{"index":6,"sense_key":"save%2:41:02::","synset_id":"deliver.v.08"},{"index":7,"sense_key":"save%2:41:05::","synset_id":"spare.v.01"},{"index":8,"sense_key":"save%2:41:00::","synset_id":"save.v.03"},{"index":9,"sense_key":"save%2:40:01::","synset_id":"save.v.05"},{"index":10,"sense_key":"save%2:30:00::","synset_id":"save.v.06"},{"index":11,"sense_key":"save%2:40:04::","synset_id":"save.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"save%2:32:04::","call_ids":["semeval2015.d001.s043.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"save%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.005815,"input_uncached_usd":0.003265,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.304410040960647}
{"item_id":"semeval2015.d001.s043.t001","gold_sense_keys":["log%1:10:02::"],"candidates":[{"index":1,"sense_key":"log%1:10:02::","synset_id":"log.n.03"},{"index":2,"sense_key":"log%1:10:01::","synset_id":"logarithm.n.01"},{"index":3,"sense_key":"log%1:06:01::","synset_id":"log.n.05"},{"index":4,"sense_key":"log%1:10:00::","synset_id":"log.n.04"},{"index":5,"sense_key":"log%1:27:01::","synset_id":"log.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"log%1:10:02::","call_ids":["semeval2015.d001.s043.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"log%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":172,"reasoning_output_tokens":156},"cost":{"currency":"USD","total_usd":0.007205000000000001,"input_uncached_usd":0.0020450000000000004,"input_cached_usd":0.0,"output_usd":0.0051600000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9443511670688167}
{"item_id":"semeval2015.d001.s044.t000","gold_sense_keys":["save%2:32:04::"],"candidates":[{"index":1,"sense_key":"save%2:41:00::","synset_id":"save.v.03"},{"index":2,"sense_key":"save%2:40:00::","synset_id":"save.v.02"},{"index":3,"sense_key":"save%2:41:05::","synset_id":"spare.v.01"},{"index":4,"sense_key":"save%2:40:03::","synset_id":"keep_open.v.01"},{"index":5,"sense_key":"save%2:40:02::","synset_id":"save.v.04"},{"index":6,"sense_key":"save%2:32:04::","synset_id":"write.v.08"},{"index":7,"sense_key":"save%2:41:02::","synset_id":"deliver.v.08"},{"index":8,"sense_key":"save%2:41:01::","synset_id":"salvage.v.01"},{"index":9,"sense_key":"save%2:40:01::","synset_id":"save.v.05"},{"index":10,"sense_key":"save%2:40:04::","synset_id":"save.v.09"},{"index":11,"sense_key":"save%2:30:00::","synset_id":"save.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"save%2:32:04::","call_ids":["semeval2015.d001.s044.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"save%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.0055450000000000004,"input_uncached_usd":0.0032350000000000005,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0315827500307932}
{"item_id":"semeval2015.d001.s044.t001","gold_sense_keys":["log%1:10:02::"],"candidates":[{"index":1,"sense_key":"log%1:06:01::","synset_id":"log.n.05"},{"index":2,"sense_key":"log%1:10:01::","synset_id":"logarithm.n.01"},{"index":3,"sense_key":"log%1:27:01::","synset_id":"log.n.01"},{"index":4,"sense_key":"log%1:10:02::","synset_id":"log.n.03"},{"index":5,"sense_key":"log%1:10:00::","synset_id":"log.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"log%1:10:02::","call_ids":["semeval2015.d001.s044.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"log%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":109},"cost":{"currency":"USD","total_usd":0.0057350000000000005,"input_uncached_usd":0.0020150000000000003,"input_cached_usd":0.0,"output_usd":0.00372,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2560525410808623}
{"item_id":"semeval2015.d001.s044.t002","gold_sense_keys":["result%1:10:00::","result%1:11:00::"],"candidates":[{"index":1,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"},{"index":2,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"result%1:11:00::","call_ids":["semeval2015.d001.s044.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"result%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":436,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":124},"cost":{"currency":"USD","total_usd":0.00638,"input_uncached_usd":0.00218,"input_cached_usd":0.0,"output_usd":0.0042,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.333420041948557}
{"item_id":"semeval2015.d001.s044.t004","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"},{"index":2,"sense_key":"able%3:00:00::","synset_id":"able.a.01"},{"index":3,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"},{"index":4,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"able%3:00:00::","call_ids":["semeval2015.d001.s044.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"able%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":413,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":95},"cost":{"currency":"USD","total_usd":0.0053950000000000005,"input_uncached_usd":0.002065,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5150448749773204}
{"item_id":"semeval2015.d001.s044.t005","gold_sense_keys":["print%2:36:00::"],"candidates":[{"index":1,"sense_key":"print%2:36:02::","synset_id":"print.v.01"},{"index":2,"sense_key":"print%2:36:01::","synset_id":"print.v.03"},{"index":3,"sense_key":"print%2:36:00::","synset_id":"print.v.04"},{"index":4,"sense_key":"print%2:36:03::","synset_id":"print.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"print%2:36:00::","call_ids":["semeval2015.d001.s044.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"print%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":349,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":115},"cost":{"currency":"USD","total_usd":0.005675,"input_uncached_usd":0.0017450000000000002,"input_cached_usd":0.0,"output_usd":0.00393,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6559583329362795}
{"item_id":"semeval2015.d001.s044.t006","gold_sense_keys":["publish%2:32:00::"],"candidates":[{"index":1,"sense_key":"publish%2:32:00::","synset_id":"publish.v.02"},{"index":2,"sense_key":"publish%2:36:00::","synset_id":"publish.v.03"},{"index":3,"sense_key":"publish%2:36:01::","synset_id":"print.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"publish%2:32:00::","call_ids":["semeval2015.d001.s044.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"publish%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":369,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":136},"cost":{"currency":"USD","total_usd":0.006405000000000001,"input_uncached_usd":0.001845,"input_cached_usd":0.0,"output_usd":0.00456,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.726275666966103}
{"item_id":"semeval2015.d001.s045.t000","gold_sense_keys":["quit%2:42:04::"],"candidates":[{"index":1,"sense_key":"quit%2:41:00::","synset_id":"leave_office.v.01"},{"index":2,"sense_key":"quit%2:38:00::","synset_id":"depart.v.04"},{"index":3,"sense_key":"quit%2:33:05::","synset_id":"drop_out.v.01"},{"index":4,"sense_key":"quit%2:42:04::","synset_id":"discontinue.v.01"},{"index":5,"sense_key":"quit%2:40:00::","synset_id":"foreswear.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"quit%2:42:04::","call_ids":["semeval2015.d001.s045.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"quit%2:42:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":232,"reasoning_output_tokens":217},"cost":{"currency":"USD","total_usd":0.00928,"input_uncached_usd":0.00232,"input_cached_usd":0.0,"output_usd":0.00696,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.150721041020006}
{"item_id":"semeval2015.d001.s046.t000","gold_sense_keys":["program%1:10:02::"],"candidates":[{"index":1,"sense_key":"program%1:10:04::","synset_id":"platform.n.02"},{"index":2,"sense_key":"program%1:10:01::","synset_id":"broadcast.n.02"},{"index":3,"sense_key":"program%1:04:00::","synset_id":"program.n.08"},{"index":4,"sense_key":"program%1:09:00::","synset_id":"plan.n.01"},{"index":5,"sense_key":"program%1:10:05::","synset_id":"course_of_study.n.01"},{"index":6,"sense_key":"program%1:10:00::","synset_id":"program.n.05"},{"index":7,"sense_key":"program%1:09:01::","synset_id":"program.n.02"},{"index":8,"sense_key":"program%1:10:02::","synset_id":"program.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"program%1:10:02::","call_ids":["semeval2015.d001.s046.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"program%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.004845,"input_uncached_usd":0.0032250000000000004,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4481860840460286}
{"item_id":"semeval2015.d001.s047.t000","gold_sense_keys":["add%2:30:00::"],"candidates":[{"index":1,"sense_key":"add%2:32:01::","synset_id":"add.v.02"},{"index":2,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":3,"sense_key":"add%2:32:00::","synset_id":"total.v.02"},{"index":4,"sense_key":"add%2:42:00::","synset_id":"add.v.06"},{"index":5,"sense_key":"add%2:31:00::","synset_id":"add.v.04"},{"index":6,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"add%2:30:00::","call_ids":["semeval2015.d001.s047.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"add%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":121},"cost":{"currency":"USD","total_usd":0.006775,"input_uncached_usd":0.0026650000000000003,"input_cached_usd":0.0,"output_usd":0.00411,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.452853334019892}
{"item_id":"semeval2015.d001.s047.t001","gold_sense_keys":["new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":2,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":3,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":6,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":7,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":8,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":9,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":10,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"},{"index":11,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"new%3:00:00::","call_ids":["semeval2015.d001.s047.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"new%3:00:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":248,"reasoning_output_tokens":232},"cost":{"currency":"USD","total_usd":0.011130000000000001,"input_uncached_usd":0.00369,"input_cached_usd":0.0,"output_usd":0.00744,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.210340458084829}
{"item_id":"semeval2015.d001.s047.t003","gold_sense_keys":["do%2:36:01::","do%2:41:04::"],"candidates":[{"index":1,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":2,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":3,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":4,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":5,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":6,"sense_key":"do%2:38:00::","synset_id":"do.v.13"},{"index":7,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":8,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":9,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":10,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":11,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":12,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":13,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"do%2:36:01::","call_ids":["semeval2015.d001.s047.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":851,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":92},"cost":{"currency":"USD","total_usd":0.007495,"input_uncached_usd":0.004255,"input_cached_usd":0.0,"output_usd":0.0032400000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7797842089785263}
{"item_id":"semeval2015.d001.s047.t004","gold_sense_keys":["go%2:38:00::"],"candidates":[{"index":1,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":2,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":3,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":4,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"},{"index":5,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":6,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"},{"index":7,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":8,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":9,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":10,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":11,"sense_key":"go%2:42:08::","synset_id":"go.v.25"},{"index":12,"sense_key":"go%2:30:08::","synset_id":"go.v.09"},{"index":13,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":14,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":15,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":16,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":17,"sense_key":"go%2:38:01::","synset_id":"go.v.03"},{"index":18,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":19,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":20,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":21,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":22,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":23,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":24,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":25,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":26,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"},{"index":27,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":28,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":29,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":30,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":21,"chosen_sense_key":"go%2:38:00::","call_ids":["semeval2015.d001.s047.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":21,"predicted_sense_key":"go%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1652,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":105},"cost":{"currency":"USD","total_usd":0.011859999999999999,"input_uncached_usd":0.00826,"input_cached_usd":0.0,"output_usd":0.0036,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1737689999863505}
{"item_id":"semeval2015.d001.s047.t006","gold_sense_keys":["add%2:30:00::"],"candidates":[{"index":1,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":2,"sense_key":"add%2:32:00::","synset_id":"total.v.02"},{"index":3,"sense_key":"add%2:42:00::","synset_id":"add.v.06"},{"index":4,"sense_key":"add%2:31:00::","synset_id":"add.v.04"},{"index":5,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"},{"index":6,"sense_key":"add%2:32:01::","synset_id":"add.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"add%2:30:00::","call_ids":["semeval2015.d001.s047.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"add%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":118},"cost":{"currency":"USD","total_usd":0.00668,"input_uncached_usd":0.00266,"input_cached_usd":0.0,"output_usd":0.00402,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8840265419567004}
{"item_id":"semeval2015.d001.s047.t007","gold_sense_keys":["add%2:30:00::"],"candidates":[{"index":1,"sense_key":"add%2:32:00::","synset_id":"total.v.02"},{"index":2,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"},{"index":3,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":4,"sense_key":"add%2:42:00::","synset_id":"add.v.06"},{"index":5,"sense_key":"add%2:31:00::","synset_id":"add.v.04"},{"index":6,"sense_key":"add%2:32:01::","synset_id":"add.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"add%2:30:00::","call_ids":["semeval2015.d001.s047.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"add%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.0053950000000000005,"input_uncached_usd":0.0026650000000000003,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9529424170032144}
{"item_id":"semeval2015.d001.s047.t008","gold_sense_keys":["new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":2,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":5,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":6,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":7,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":8,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":9,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":10,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":11,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"new%3:00:00::","call_ids":["semeval2015.d001.s047.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"new%3:00:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":168,"reasoning_output_tokens":152},"cost":{"currency":"USD","total_usd":0.00873,"input_uncached_usd":0.00369,"input_cached_usd":0.0,"output_usd":0.00504,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6331702090101317}
{"item_id":"semeval2015.d001.s047.t009","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":7,"sense_key":"function%1:24:00::","synset_id":"function.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"function%1:24:00::","call_ids":["semeval2015.d001.s047.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"function%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.004845,"input_uncached_usd":0.002955,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3516452920157462}
{"item_id":"semeval2015.d001.s048.t000","gold_sense_keys":["then%4:02:00::"],"candidates":[{"index":1,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":2,"sense_key":"then%4:02:01::","synset_id":"then.r.03"},{"index":3,"sense_key":"then%4:02:02::","synset_id":"then.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"then%4:02:00::","call_ids":["semeval2015.d001.s048.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"then%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.0039250000000000005,"input_uncached_usd":0.0019150000000000003,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6678201250033453}
{"item_id":"semeval2015.d001.s048.t001","gold_sense_keys":["focus%1:09:00::"],"candidates":[{"index":1,"sense_key":"focus%1:26:01::","synset_id":"stress.n.03"},{"index":2,"sense_key":"focus%1:19:00::","synset_id":"focus.n.06"},{"index":3,"sense_key":"focus%1:09:00::","synset_id":"focus.n.01"},{"index":4,"sense_key":"focus%1:07:03::","synset_id":"focus.n.03"},{"index":5,"sense_key":"focus%1:26:00::","synset_id":"focus.n.04"},{"index":6,"sense_key":"focus%1:15:00::","synset_id":"focus.n.07"},{"index":7,"sense_key":"focus%1:07:01::","synset_id":"focus.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"focus%1:09:00::","call_ids":["semeval2015.d001.s048.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"focus%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":540,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":98},"cost":{"currency":"USD","total_usd":0.00609,"input_uncached_usd":0.0027,"input_cached_usd":0.0,"output_usd":0.0033900000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4175679580075666}
{"item_id":"semeval2015.d001.s048.t003","gold_sense_keys":["text%1:10:00::"],"candidates":[{"index":1,"sense_key":"text%1:10:03::","synset_id":"text.n.02"},{"index":2,"sense_key":"text%1:10:00::","synset_id":"text.n.01"},{"index":3,"sense_key":"text%1:10:02::","synset_id":"textbook.n.01"},{"index":4,"sense_key":"text%1:10:01::","synset_id":"text.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"text%1:10:00::","call_ids":["semeval2015.d001.s048.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"text%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.004745,"input_uncached_usd":0.002255,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.971982334041968}
{"item_id":"semeval2015.d001.s048.t005","gold_sense_keys":["type%2:32:00::"],"candidates":[{"index":1,"sense_key":"type%2:31:00::","synset_id":"type.v.02"},{"index":2,"sense_key":"type%2:32:00::","synset_id":"type.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"type%2:32:00::","call_ids":["semeval2015.d001.s048.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"type%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":331,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.003305,"input_uncached_usd":0.0016550000000000002,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.275308542069979}
{"item_id":"semeval2015.d001.s048.t006","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":2,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":3,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":4,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":5,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"},{"index":6,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":7,"sense_key":"function%1:14:00::","synset_id":"function.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"function%1:24:00::","call_ids":["semeval2015.d001.s048.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"function%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004995,"input_uncached_usd":0.002955,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4760858749505132}
{"item_id":"semeval2015.d001.s049.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":2,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"want%2:37:00::","call_ids":["semeval2015.d001.s049.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":468,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.0049499999999999995,"input_uncached_usd":0.00234,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0134269590489566}
{"item_id":"semeval2015.d001.s049.t001","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":2,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":3,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":4,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":5,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":6,"sense_key":"use%2:41:14::","synset_id":"use.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s049.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":111},"cost":{"currency":"USD","total_usd":0.006575,"input_uncached_usd":0.002765,"input_cached_usd":0.0,"output_usd":0.00381,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.675229375017807}
{"item_id":"semeval2015.d001.s049.t002","gold_sense_keys":["typical%3:00:00::","typical%5:00:00:normal:01"],"candidates":[{"index":1,"sense_key":"typical%3:00:00::","synset_id":"typical.a.01"},{"index":2,"sense_key":"typical%5:00:00:normal:01","synset_id":"typical.s.02"},{"index":3,"sense_key":"typical%5:00:00:characteristic:00","synset_id":"distinctive.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"typical%5:00:00:normal:01","call_ids":["semeval2015.d001.s049.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"typical%5:00:00:normal:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":496,"reasoning_output_tokens":480},"cost":{"currency":"USD","total_usd":0.01696,"input_uncached_usd":0.0020800000000000003,"input_cached_usd":0.0,"output_usd":0.01488,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.061434499919415}
{"item_id":"semeval2015.d001.s049.t004","gold_sense_keys":["necessary%3:00:00::"],"candidates":[{"index":1,"sense_key":"necessary%3:00:00::","synset_id":"necessary.a.01"},{"index":2,"sense_key":"necessary%5:00:00:inevitable:00","synset_id":"necessary.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"necessary%3:00:00::","call_ids":["semeval2015.d001.s049.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"necessary%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":329,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.004405,"input_uncached_usd":0.0016450000000000002,"input_cached_usd":0.0,"output_usd":0.00276,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1918354999506846}
{"item_id":"semeval2015.d001.s049.t005","gold_sense_keys":["specify%2:32:02::"],"candidates":[{"index":1,"sense_key":"specify%2:32:02::","synset_id":"specify.v.02"},{"index":2,"sense_key":"specify%2:32:03::","synset_id":"stipulate.v.01"},{"index":3,"sense_key":"specify%2:42:00::","synset_id":"specify.v.03"},{"index":4,"sense_key":"specify%2:31:00::","synset_id":"intend.v.02"},{"index":5,"sense_key":"specify%2:31:05::","synset_id":"assign.v.04"},{"index":6,"sense_key":"specify%2:32:00::","synset_id":"specify.v.04"},{"index":7,"sense_key":"specify%2:31:02::","synset_id":"pin_down.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"specify%2:32:02::","call_ids":["semeval2015.d001.s049.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"specify%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":4476,"reasoning_output_tokens":4412},"cost":{"currency":"USD","total_usd":0.13735,"input_uncached_usd":0.0030700000000000002,"input_cached_usd":0.0,"output_usd":0.13428,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":63.225484708091244}
{"item_id":"semeval2015.d001.s050.t001","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":6,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"},{"index":7,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"function%1:24:00::","call_ids":["semeval2015.d001.s050.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"function%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":600,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00489,"input_uncached_usd":0.003,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.411700165946968}
{"item_id":"semeval2015.d001.s050.t003","gold_sense_keys":["display%2:39:00::"],"candidates":[{"index":1,"sense_key":"display%2:39:00::","synset_id":"expose.v.03"},{"index":2,"sense_key":"display%2:41:00::","synset_id":"display.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"display%2:39:00::","call_ids":["semeval2015.d001.s050.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"display%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":349,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.003365,"input_uncached_usd":0.0017450000000000002,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4685267499880865}
{"item_id":"semeval2015.d001.s050.t005","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"},{"index":2,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":3,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["semeval2015.d001.s050.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":35},"cost":{"currency":"USD","total_usd":0.00355,"input_uncached_usd":0.00202,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6431580829666927}
{"item_id":"semeval2015.d001.s050.t006","gold_sense_keys":["window%1:06:03::"],"candidates":[{"index":1,"sense_key":"window%1:06:03::","synset_id":"window.n.08"},{"index":2,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":3,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":4,"sense_key":"window%1:06:01::","synset_id":"window.n.02"},{"index":5,"sense_key":"window%1:06:00::","synset_id":"window.n.01"},{"index":6,"sense_key":"window%1:17:00::","synset_id":"window.n.04"},{"index":7,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":8,"sense_key":"window%1:28:00::","synset_id":"window.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"window%1:06:03::","call_ids":["semeval2015.d001.s050.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"window%1:06:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":590,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.00451,"input_uncached_usd":0.0029500000000000004,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8312201659427956}
{"item_id":"semeval2015.d001.s051.t000","gold_sense_keys":["set%2:35:00::"],"candidates":[{"index":1,"sense_key":"set%2:36:12::","synset_id":"set.v.13"},{"index":2,"sense_key":"set%2:30:00::","synset_id":"adjust.v.01"},{"index":3,"sense_key":"set%2:36:01::","synset_id":"set.v.09"},{"index":4,"sense_key":"set%2:36:11::","synset_id":"typeset.v.01"},{"index":5,"sense_key":"set%2:35:15::","synset_id":"set.v.16"},{"index":6,"sense_key":"set%2:30:12::","synset_id":"set_up.v.04"},{"index":7,"sense_key":"set%2:35:09::","synset_id":"plant.v.01"},{"index":8,"sense_key":"set%2:36:00::","synset_id":"arrange.v.06"},{"index":9,"sense_key":"set%2:30:07::","synset_id":"jell.v.01"},{"index":10,"sense_key":"set%2:38:00::","synset_id":"set.v.10"},{"index":11,"sense_key":"set%2:30:02::","synset_id":"set.v.08"},{"index":12,"sense_key":"set%2:33:00::","synset_id":"sic.v.01"},{"index":13,"sense_key":"set%2:31:13::","synset_id":"place.v.11"},{"index":14,"sense_key":"set%2:32:01::","synset_id":"set.v.04"},{"index":15,"sense_key":"set%2:30:01::","synset_id":"set.v.05"},{"index":16,"sense_key":"set%2:35:00::","synset_id":"put.v.01"},{"index":17,"sense_key":"set%2:30:06::","synset_id":"rig.v.04"},{"index":18,"sense_key":"set%2:29:05::","synset_id":"dress.v.16"},{"index":19,"sense_key":"set%2:35:05::","synset_id":"set.v.17"},{"index":20,"sense_key":"set%2:31:04::","synset_id":"determine.v.03"},{"index":21,"sense_key":"set%2:35:02::","synset_id":"set.v.06"},{"index":22,"sense_key":"set%2:30:09::","synset_id":"fix.v.12"},{"index":23,"sense_key":"set%2:35:04::","synset_id":"set.v.18"},{"index":24,"sense_key":"set%2:29:06::","synset_id":"fructify.v.03"},{"index":25,"sense_key":"set%2:32:00::","synset_id":"specify.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":16,"chosen_sense_key":"set%2:35:00::","call_ids":["semeval2015.d001.s051.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":16,"predicted_sense_key":"set%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1238,"cached_input_tokens":0,"output_tokens":181,"reasoning_output_tokens":165},"cost":{"currency":"USD","total_usd":0.011620000000000002,"input_uncached_usd":0.00619,"input_cached_usd":0.0,"output_usd":0.00543,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1626921669812873}
{"item_id":"semeval2015.d001.s051.t001","gold_sense_keys":["several%5:00:00:some:00"],"candidates":[{"index":1,"sense_key":"several%5:00:00:different:00","synset_id":"several.s.03"},{"index":2,"sense_key":"several%5:00:00:some:00","synset_id":"several.s.01"},{"index":3,"sense_key":"several%5:00:00:individual:00","synset_id":"respective.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"several%5:00:00:some:00","call_ids":["semeval2015.d001.s051.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"several%5:00:00:some:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.0038150000000000002,"input_uncached_usd":0.0020150000000000003,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.257484458037652}
{"item_id":"semeval2015.d001.s051.t003","gold_sense_keys":["same%3:00:02::"],"candidates":[{"index":1,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":2,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"},{"index":3,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":4,"sense_key":"same%3:00:04::","synset_id":"like.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"same%3:00:02::","call_ids":["semeval2015.d001.s051.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"same%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":58},"cost":{"currency":"USD","total_usd":0.004355,"input_uncached_usd":0.0021650000000000003,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8831224169116467}
{"item_id":"semeval2015.d001.s052.t000","gold_sense_keys":["just%4:02:00::"],"candidates":[{"index":1,"sense_key":"just%4:02:03::","synset_id":"just.r.04"},{"index":2,"sense_key":"just%4:02:05::","synset_id":"precisely.r.01"},{"index":3,"sense_key":"just%4:02:04::","synset_id":"barely.r.01"},{"index":4,"sense_key":"just%4:02:00::","synset_id":"merely.r.01"},{"index":5,"sense_key":"just%4:02:06::","synset_id":"just.r.06"},{"index":6,"sense_key":"just%4:02:01::","synset_id":"just.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"just%4:02:00::","call_ids":["semeval2015.d001.s052.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"just%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.005935,"input_uncached_usd":0.0028150000000000002,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.154285499942489}
{"item_id":"semeval2015.d001.s052.t001","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:34:02::","synset_id":"use.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s052.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.004790000000000001,"input_uncached_usd":0.0027800000000000004,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7918979580281302}
{"item_id":"semeval2015.d001.s052.t002","gold_sense_keys":["add%2:30:00::"],"candidates":[{"index":1,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"},{"index":2,"sense_key":"add%2:32:00::","synset_id":"total.v.02"},{"index":3,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":4,"sense_key":"add%2:32:01::","synset_id":"add.v.02"},{"index":5,"sense_key":"add%2:31:00::","synset_id":"add.v.04"},{"index":6,"sense_key":"add%2:42:00::","synset_id":"add.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"add%2:30:00::","call_ids":["semeval2015.d001.s052.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"add%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":101},"cost":{"currency":"USD","total_usd":0.006240000000000001,"input_uncached_usd":0.0027600000000000003,"input_cached_usd":0.0,"output_usd":0.00348,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3231358329067007}
{"item_id":"semeval2015.d001.s052.t003","gold_sense_keys":["list%1:10:00::"],"candidates":[{"index":1,"sense_key":"list%1:10:00::","synset_id":"list.n.01"},{"index":2,"sense_key":"list%1:07:00::","synset_id":"tilt.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"list%1:10:00::","call_ids":["semeval2015.d001.s052.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"list%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":365,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.0040750000000000005,"input_uncached_usd":0.0018250000000000002,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3143903750460595}
{"item_id":"semeval2015.d001.s052.t004","gold_sense_keys":["mode%1:26:00::"],"candidates":[{"index":1,"sense_key":"mode%1:26:00::","synset_id":"mode.n.02"},{"index":2,"sense_key":"mode%1:24:00::","synset_id":"modality.n.01"},{"index":3,"sense_key":"mode%1:09:00::","synset_id":"mode.n.06"},{"index":4,"sense_key":"mode%1:07:00::","synset_id":"manner.n.01"},{"index":5,"sense_key":"mode%1:24:02::","synset_id":"mood.n.03"},{"index":6,"sense_key":"mode%1:10:00::","synset_id":"mode.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mode%1:26:00::","call_ids":["semeval2015.d001.s052.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mode%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.004015,"input_uncached_usd":0.0024850000000000002,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4476521670585498}
{"item_id":"semeval2015.d001.s053.t000","gold_sense_keys":["set%2:32:00::"],"candidates":[{"index":1,"sense_key":"set%2:29:06::","synset_id":"fructify.v.03"},{"index":2,"sense_key":"set%2:35:15::","synset_id":"set.v.16"},{"index":3,"sense_key":"set%2:32:00::","synset_id":"specify.v.02"},{"index":4,"sense_key":"set%2:36:11::","synset_id":"typeset.v.01"},{"index":5,"sense_key":"set%2:35:00::","synset_id":"put.v.01"},{"index":6,"sense_key":"set%2:31:13::","synset_id":"place.v.11"},{"index":7,"sense_key":"set%2:30:01::","synset_id":"set.v.05"},{"index":8,"sense_key":"set%2:30:07::","synset_id":"jell.v.01"},{"index":9,"sense_key":"set%2:30:02::","synset_id":"set.v.08"},{"index":10,"sense_key":"set%2:30:12::","synset_id":"set_up.v.04"},{"index":11,"sense_key":"set%2:38:00::","synset_id":"set.v.10"},{"index":12,"sense_key":"set%2:36:12::","synset_id":"set.v.13"},{"index":13,"sense_key":"set%2:35:09::","synset_id":"plant.v.01"},{"index":14,"sense_key":"set%2:30:09::","synset_id":"fix.v.12"},{"index":15,"sense_key":"set%2:35:02::","synset_id":"set.v.06"},{"index":16,"sense_key":"set%2:30:00::","synset_id":"adjust.v.01"},{"index":17,"sense_key":"set%2:32:01::","synset_id":"set.v.04"},{"index":18,"sense_key":"set%2:31:04::","synset_id":"determine.v.03"},{"index":19,"sense_key":"set%2:36:00::","synset_id":"arrange.v.06"},{"index":20,"sense_key":"set%2:33:00::","synset_id":"sic.v.01"},{"index":21,"sense_key":"set%2:35:04::","synset_id":"set.v.18"},{"index":22,"sense_key":"set%2:35:05::","synset_id":"set.v.17"},{"index":23,"sense_key":"set%2:29:05::","synset_id":"dress.v.16"},{"index":24,"sense_key":"set%2:36:01::","synset_id":"set.v.09"},{"index":25,"sense_key":"set%2:30:06::","synset_id":"rig.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"set%2:32:00::","call_ids":["semeval2015.d001.s053.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"set%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1202,"cached_input_tokens":0,"output_tokens":251,"reasoning_output_tokens":235},"cost":{"currency":"USD","total_usd":0.01354,"input_uncached_usd":0.006010000000000001,"input_cached_usd":0.0,"output_usd":0.00753,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.248993874993175}
{"item_id":"semeval2015.d001.s053.t002","gold_sense_keys":["color%1:07:00::","color%1:07:01::"],"candidates":[{"index":1,"sense_key":"color%1:07:02::","synset_id":"color.n.03"},{"index":2,"sense_key":"color%1:27:00::","synset_id":"coloring_material.n.01"},{"index":3,"sense_key":"color%1:07:04::","synset_id":"semblance.n.01"},{"index":4,"sense_key":"color%1:07:01::","synset_id":"color.n.08"},{"index":5,"sense_key":"color%1:07:03::","synset_id":"color.n.02"},{"index":6,"sense_key":"color%1:14:00::","synset_id":"color.n.04"},{"index":7,"sense_key":"color%1:09:01::","synset_id":"color.n.07"},{"index":8,"sense_key":"color%1:07:00::","synset_id":"color.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"color%1:07:00::","call_ids":["semeval2015.d001.s053.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"color%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1023},"cost":{"currency":"USD","total_usd":0.03450500000000001,"input_uncached_usd":0.003125,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":15.316397875081748}
{"item_id":"semeval2015.d002.s000.t000","gold_sense_keys":["foundation%1:14:00::"],"candidates":[{"index":1,"sense_key":"foundation%1:06:01::","synset_id":"foundation_garment.n.01"},{"index":2,"sense_key":"foundation%1:09:01::","synset_id":"foundation.n.04"},{"index":3,"sense_key":"foundation%1:09:00::","synset_id":"basis.n.02"},{"index":4,"sense_key":"foundation%1:06:00::","synset_id":"foundation.n.03"},{"index":5,"sense_key":"foundation%1:24:00::","synset_id":"foundation.n.01"},{"index":6,"sense_key":"foundation%1:14:00::","synset_id":"foundation.n.02"},{"index":7,"sense_key":"foundation%1:04:00::","synset_id":"initiation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"foundation%1:14:00::","call_ids":["semeval2015.d002.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"foundation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.004945,"input_uncached_usd":0.002635,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6730509590124711}
{"item_id":"semeval2015.d002.s000.t002","gold_sense_keys":["involve%2:42:04::"],"candidates":[{"index":1,"sense_key":"involve%2:42:00::","synset_id":"imply.v.05"},{"index":2,"sense_key":"involve%2:31:00::","synset_id":"involve.v.06"},{"index":3,"sense_key":"involve%2:42:04::","synset_id":"involve.v.02"},{"index":4,"sense_key":"involve%2:42:02::","synset_id":"involve.v.05"},{"index":5,"sense_key":"involve%2:42:07::","synset_id":"necessitate.v.01"},{"index":6,"sense_key":"involve%2:42:01::","synset_id":"involve.v.01"},{"index":7,"sense_key":"involve%2:30:00::","synset_id":"involve.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"involve%2:42:04::","call_ids":["semeval2015.d002.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"involve%2:42:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.004205,"input_uncached_usd":0.0023450000000000003,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8962504169903696}
{"item_id":"semeval2015.d002.s000.t003","gold_sense_keys":["organise%2:36:00::","organise%2:41:00::"],"candidates":[{"index":1,"sense_key":"organise%2:41:01::","synset_id":"form.v.01"},{"index":2,"sense_key":"organise%2:30:00::","synset_id":"organize.v.04"},{"index":3,"sense_key":"organise%2:41:02::","synset_id":"unionize.v.02"},{"index":4,"sense_key":"organise%2:41:00::","synset_id":"organize.v.02"},{"index":5,"sense_key":"organise%2:36:00::","synset_id":"organize.v.05"},{"index":6,"sense_key":"organise%2:31:00::","synset_id":"mastermind.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"organise%2:36:00::","call_ids":["semeval2015.d002.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"organise%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":86},"cost":{"currency":"USD","total_usd":0.00537,"input_uncached_usd":0.00231,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2574462910415605}
{"item_id":"semeval2015.d002.s000.t004","gold_sense_keys":["conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"},{"index":2,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":3,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conference%1:14:00::","call_ids":["semeval2015.d002.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":304,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.00431,"input_uncached_usd":0.00152,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0618266250239685}
{"item_id":"semeval2015.d002.s000.t005","gold_sense_keys":["deal%2:32:08::"],"candidates":[{"index":1,"sense_key":"deal%2:40:07::","synset_id":"share.v.04"},{"index":2,"sense_key":"deal%2:40:11::","synset_id":"deal.v.13"},{"index":3,"sense_key":"deal%2:41:13::","synset_id":"manage.v.02"},{"index":4,"sense_key":"deal%2:40:00::","synset_id":"deal.v.06"},{"index":5,"sense_key":"deal%2:41:09::","synset_id":"deal.v.03"},{"index":6,"sense_key":"deal%2:41:02::","synset_id":"conduct.v.01"},{"index":7,"sense_key":"deal%2:40:06::","synset_id":"deal.v.12"},{"index":8,"sense_key":"deal%2:32:08::","synset_id":"cover.v.05"},{"index":9,"sense_key":"deal%2:40:02::","synset_id":"deal.v.09"},{"index":10,"sense_key":"deal%2:41:04::","synset_id":"deal.v.08"},{"index":11,"sense_key":"deal%2:41:00::","synset_id":"cope.v.01"},{"index":12,"sense_key":"deal%2:31:10::","synset_id":"consider.v.03"},{"index":13,"sense_key":"deal%2:40:01::","synset_id":"distribute.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"deal%2:32:08::","call_ids":["semeval2015.d002.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"deal%2:32:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":780,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":66},"cost":{"currency":"USD","total_usd":0.00636,"input_uncached_usd":0.0039000000000000003,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.890133292065002}
{"item_id":"semeval2015.d002.s000.t006","gold_sense_keys":["critical%5:00:00:crucial:00"],"candidates":[{"index":1,"sense_key":"critical%5:00:00:crucial:00","synset_id":"critical.s.02"},{"index":2,"sense_key":"critical%3:01:00::","synset_id":"critical.a.07"},{"index":3,"sense_key":"critical%3:00:03::","synset_id":"critical.a.06"},{"index":4,"sense_key":"critical%3:00:04::","synset_id":"critical.a.02"},{"index":5,"sense_key":"critical%5:00:00:indispensable:00","synset_id":"critical.s.01"},{"index":6,"sense_key":"critical%3:00:01::","synset_id":"critical.a.01"},{"index":7,"sense_key":"critical%3:00:02::","synset_id":"critical.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"critical%5:00:00:crucial:00","call_ids":["semeval2015.d002.s000.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"critical%5:00:00:crucial:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01834,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.112490416970104}
{"item_id":"semeval2015.d002.s000.t007","gold_sense_keys":["challenge%1:26:00::"],"candidates":[{"index":1,"sense_key":"challenge%1:10:03::","synset_id":"challenge.n.05"},{"index":2,"sense_key":"challenge%1:26:00::","synset_id":"challenge.n.01"},{"index":3,"sense_key":"challenge%1:10:00::","synset_id":"challenge.n.02"},{"index":4,"sense_key":"challenge%1:10:02::","synset_id":"challenge.n.04"},{"index":5,"sense_key":"challenge%1:10:01::","synset_id":"challenge.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"challenge%1:26:00::","call_ids":["semeval2015.d002.s000.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"challenge%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":356,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.00367,"input_uncached_usd":0.0017800000000000001,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.147179750027135}
{"item_id":"semeval2015.d002.s001.t000","gold_sense_keys":["participant%1:18:00::"],"candidates":[{"index":1,"sense_key":"participant%1:18:00::","synset_id":"participant.n.01"},{"index":2,"sense_key":"participant%1:18:01::","synset_id":"player.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"participant%1:18:00::","call_ids":["semeval2015.d002.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"participant%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":284,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.0030099999999999997,"input_uncached_usd":0.00142,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5565211250213906}
{"item_id":"semeval2015.d002.s001.t001","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":2,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"},{"index":3,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["semeval2015.d002.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":371,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.003265,"input_uncached_usd":0.001855,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4100582910468802}
{"item_id":"semeval2015.d002.s001.t003","gold_sense_keys":["attend%2:42:00::"],"candidates":[{"index":1,"sense_key":"attend%2:42:01::","synset_id":"attend.v.03"},{"index":2,"sense_key":"attend%2:42:00::","synset_id":"attend.v.01"},{"index":3,"sense_key":"attend%2:41:00::","synset_id":"serve.v.10"},{"index":4,"sense_key":"attend%2:41:12::","synset_id":"attend.v.02"},{"index":5,"sense_key":"attend%2:39:00::","synset_id":"attend.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"attend%2:42:00::","call_ids":["semeval2015.d002.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"attend%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.00413,"input_uncached_usd":0.0024200000000000003,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4415746659506112}
{"item_id":"semeval2015.d002.s001.t004","gold_sense_keys":["debate%2:32:00::"],"candidates":[{"index":1,"sense_key":"debate%2:32:00::","synset_id":"debate.v.03"},{"index":2,"sense_key":"debate%2:32:01::","synset_id":"argue.v.02"},{"index":3,"sense_key":"debate%2:31:00::","synset_id":"debate.v.01"},{"index":4,"sense_key":"debate%2:32:02::","synset_id":"consider.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"debate%2:32:00::","call_ids":["semeval2015.d002.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"debate%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":176,"reasoning_output_tokens":160},"cost":{"currency":"USD","total_usd":0.007195000000000001,"input_uncached_usd":0.0019150000000000003,"input_cached_usd":0.0,"output_usd":0.00528,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.448716082959436}
{"item_id":"semeval2015.d002.s001.t005","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":4,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":5,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":6,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":7,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":10,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":11,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"issue%1:09:01::","call_ids":["semeval2015.d002.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":772,"cached_input_tokens":0,"output_tokens":299,"reasoning_output_tokens":283},"cost":{"currency":"USD","total_usd":0.012830000000000001,"input_uncached_usd":0.00386,"input_cached_usd":0.0,"output_usd":0.00897,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.89033854205627}
{"item_id":"semeval2015.d002.s001.t008","gold_sense_keys":["actor%1:18:01::"],"candidates":[{"index":1,"sense_key":"actor%1:18:01::","synset_id":"actor.n.02"},{"index":2,"sense_key":"actor%1:18:00::","synset_id":"actor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"actor%1:18:01::","call_ids":["semeval2015.d002.s001.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"actor%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":314,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.0034000000000000002,"input_uncached_usd":0.0015700000000000002,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6328871669247746}
{"item_id":"semeval2015.d002.s001.t011","gold_sense_keys":["authority%1:18:01::"],"candidates":[{"index":1,"sense_key":"authority%1:10:00::","synset_id":"authority.n.07"},{"index":2,"sense_key":"authority%1:14:00::","synset_id":"agency.n.01"},{"index":3,"sense_key":"authority%1:07:00::","synset_id":"authority.n.01"},{"index":4,"sense_key":"authority%1:09:00::","synset_id":"assurance.n.01"},{"index":5,"sense_key":"authority%1:07:02::","synset_id":"authority.n.06"},{"index":6,"sense_key":"authority%1:18:00::","synset_id":"authority.n.03"},{"index":7,"sense_key":"authority%1:18:01::","synset_id":"authority.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"authority%1:14:00::","call_ids":["semeval2015.d002.s001.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"authority%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":240,"reasoning_output_tokens":224},"cost":{"currency":"USD","total_usd":0.009955,"input_uncached_usd":0.002755,"input_cached_usd":0.0,"output_usd":0.0072,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.1816201250767335}
{"item_id":"semeval2015.d002.s001.t012","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"},{"index":2,"sense_key":"social%3:00:00::","synset_id":"social.a.02"},{"index":3,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":4,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"},{"index":5,"sense_key":"social%3:01:00::","synset_id":"social.a.01"},{"index":6,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s001.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":130},"cost":{"currency":"USD","total_usd":0.006750000000000001,"input_uncached_usd":0.0024000000000000002,"input_cached_usd":0.0,"output_usd":0.00435,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.029798957984895}
{"item_id":"semeval2015.d002.s001.t013","gold_sense_keys":["voluntary%3:00:01::"],"candidates":[{"index":1,"sense_key":"voluntary%3:00:01::","synset_id":"voluntary.a.01"},{"index":2,"sense_key":"voluntary%3:00:02::","synset_id":"voluntary.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"voluntary%3:00:01::","call_ids":["semeval2015.d002.s001.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"voluntary%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":308,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":105},"cost":{"currency":"USD","total_usd":0.0051400000000000005,"input_uncached_usd":0.0015400000000000001,"input_cached_usd":0.0,"output_usd":0.0036,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.284123416990042}
{"item_id":"semeval2015.d002.s001.t014","gold_sense_keys":["community%1:14:00::"],"candidates":[{"index":1,"sense_key":"community%1:26:00::","synset_id":"community.n.04"},{"index":2,"sense_key":"community%1:14:00::","synset_id":"community.n.01"},{"index":3,"sense_key":"community%1:14:02::","synset_id":"community.n.06"},{"index":4,"sense_key":"community%1:14:04::","synset_id":"community.n.03"},{"index":5,"sense_key":"community%1:21:00::","synset_id":"community.n.02"},{"index":6,"sense_key":"community%1:15:00::","synset_id":"residential_district.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"community%1:14:00::","call_ids":["semeval2015.d002.s001.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"community%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":159,"reasoning_output_tokens":143},"cost":{"currency":"USD","total_usd":0.007020000000000001,"input_uncached_usd":0.0022500000000000003,"input_cached_usd":0.0,"output_usd":0.00477,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.915491916006431}
{"item_id":"semeval2015.d002.s001.t015","gold_sense_keys":["organisation%1:14:00::","organisation%1:14:01::"],"candidates":[{"index":1,"sense_key":"organisation%1:09:00::","synset_id":"arrangement.n.03"},{"index":2,"sense_key":"organisation%1:04:02::","synset_id":"constitution.n.02"},{"index":3,"sense_key":"organisation%1:14:01::","synset_id":"administration.n.02"},{"index":4,"sense_key":"organisation%1:07:00::","synset_id":"organization.n.05"},{"index":5,"sense_key":"organisation%1:04:00::","synset_id":"organization.n.04"},{"index":6,"sense_key":"organisation%1:04:01::","synset_id":"organization.n.06"},{"index":7,"sense_key":"organisation%1:14:00::","synset_id":"organization.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"organisation%1:14:00::","call_ids":["semeval2015.d002.s001.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"organisation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.00609,"input_uncached_usd":0.00303,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.488391292048618}
{"item_id":"semeval2015.d002.s001.t017","gold_sense_keys":["service%1:04:08::"],"candidates":[{"index":1,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":2,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":3,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":4,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":5,"sense_key":"service%1:04:06::","synset_id":"service.n.15"},{"index":6,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":7,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":8,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":9,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":10,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":11,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":12,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":13,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":14,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":15,"sense_key":"service%1:04:04::","synset_id":"service.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"service%1:04:08::","call_ids":["semeval2015.d002.s001.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"service%1:04:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":98},"cost":{"currency":"USD","total_usd":0.007464999999999999,"input_uncached_usd":0.004045,"input_cached_usd":0.0,"output_usd":0.0034200000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1828284170478582}
{"item_id":"semeval2015.d002.s001.t018","gold_sense_keys":["user%1:18:00::"],"candidates":[{"index":1,"sense_key":"user%1:18:01::","synset_id":"exploiter.n.01"},{"index":2,"sense_key":"user%1:18:02::","synset_id":"drug_user.n.01"},{"index":3,"sense_key":"user%1:18:00::","synset_id":"user.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"user%1:18:00::","call_ids":["semeval2015.d002.s001.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"user%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":321,"cached_input_tokens":0,"output_tokens":30,"reasoning_output_tokens":14},"cost":{"currency":"USD","total_usd":0.0025050000000000003,"input_uncached_usd":0.001605,"input_cached_usd":0.0,"output_usd":0.0009,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.1911622079787776}
{"item_id":"semeval2015.d002.s002.t002","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":2,"sense_key":"case%1:06:01::","synset_id":"case.n.20"},{"index":3,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":4,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":7,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":8,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":9,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":10,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":11,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":12,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":13,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":18,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":19,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":20,"sense_key":"case%1:11:00::","synset_id":"case.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":20,"chosen_sense_key":"case%1:11:00::","call_ids":["semeval2015.d002.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":20,"predicted_sense_key":"case%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1106,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":101},"cost":{"currency":"USD","total_usd":0.00901,"input_uncached_usd":0.00553,"input_cached_usd":0.0,"output_usd":0.00348,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.219415875035338}
{"item_id":"semeval2015.d002.s002.t003","gold_sense_keys":["report%2:32:04::"],"candidates":[{"index":1,"sense_key":"report%2:32:04::","synset_id":"report.v.02"},{"index":2,"sense_key":"report%2:32:00::","synset_id":"report.v.01"},{"index":3,"sense_key":"report%2:32:03::","synset_id":"report.v.03"},{"index":4,"sense_key":"report%2:32:13::","synset_id":"report.v.05"},{"index":5,"sense_key":"report%2:32:01::","synset_id":"report.v.04"},{"index":6,"sense_key":"report%2:32:02::","synset_id":"report.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%2:32:04::","call_ids":["semeval2015.d002.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":213,"reasoning_output_tokens":198},"cost":{"currency":"USD","total_usd":0.008885,"input_uncached_usd":0.0024950000000000003,"input_cached_usd":0.0,"output_usd":0.00639,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.711684333975427}
{"item_id":"semeval2015.d002.s002.t004","gold_sense_keys":["attempt%1:04:00::"],"candidates":[{"index":1,"sense_key":"attempt%1:04:00::","synset_id":"attempt.n.01"},{"index":2,"sense_key":"attempt%1:04:02::","synset_id":"attack.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attempt%1:04:00::","call_ids":["semeval2015.d002.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attempt%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":346,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.0032900000000000004,"input_uncached_usd":0.0017300000000000002,"input_cached_usd":0.0,"output_usd":0.00156,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.482270915992558}
{"item_id":"semeval2015.d002.s002.t005","gold_sense_keys":["involve%2:42:04::"],"candidates":[{"index":1,"sense_key":"involve%2:42:07::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"involve%2:42:00::","synset_id":"imply.v.05"},{"index":3,"sense_key":"involve%2:31:00::","synset_id":"involve.v.06"},{"index":4,"sense_key":"involve%2:42:02::","synset_id":"involve.v.05"},{"index":5,"sense_key":"involve%2:42:04::","synset_id":"involve.v.02"},{"index":6,"sense_key":"involve%2:42:01::","synset_id":"involve.v.01"},{"index":7,"sense_key":"involve%2:30:00::","synset_id":"involve.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"involve%2:42:04::","call_ids":["semeval2015.d002.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"involve%2:42:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.004355,"input_uncached_usd":0.0025550000000000004,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.008661625091918}
{"item_id":"semeval2015.d002.s002.t006","gold_sense_keys":["user%1:18:00::"],"candidates":[{"index":1,"sense_key":"user%1:18:02::","synset_id":"drug_user.n.01"},{"index":2,"sense_key":"user%1:18:01::","synset_id":"exploiter.n.01"},{"index":3,"sense_key":"user%1:18:00::","synset_id":"user.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"user%1:18:00::","call_ids":["semeval2015.d002.s002.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"user%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":337,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.003155,"input_uncached_usd":0.001685,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1336852909298614}
{"item_id":"semeval2015.d002.s002.t007","gold_sense_keys":["service%1:04:08::"],"candidates":[{"index":1,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":2,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":3,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":4,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":5,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":6,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":7,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":8,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":9,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":10,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":11,"sense_key":"service%1:04:06::","synset_id":"service.n.15"},{"index":12,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":13,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":14,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":15,"sense_key":"service%1:04:07::","synset_id":"service.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"service%1:04:08::","call_ids":["semeval2015.d002.s002.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"service%1:04:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":824,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":119},"cost":{"currency":"USD","total_usd":0.00817,"input_uncached_usd":0.00412,"input_cached_usd":0.0,"output_usd":0.00405,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.017907749977894}
{"item_id":"semeval2015.d002.s002.t008","gold_sense_keys":["planning%1:04:00::","planning%1:04:02::"],"candidates":[{"index":1,"sense_key":"planning%1:04:00::","synset_id":"planning.n.01"},{"index":2,"sense_key":"planning%1:09:00::","synset_id":"planning.n.03"},{"index":3,"sense_key":"planning%1:04:02::","synset_id":"planning.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"planning%1:04:02::","call_ids":["semeval2015.d002.s002.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"planning%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":371,"cached_input_tokens":0,"output_tokens":5131,"reasoning_output_tokens":5061},"cost":{"currency":"USD","total_usd":0.15578499999999998,"input_uncached_usd":0.001855,"input_cached_usd":0.0,"output_usd":0.15393,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":86.88331504096277}
{"item_id":"semeval2015.d002.s002.t009","gold_sense_keys":["involvement%1:04:00::"],"candidates":[{"index":1,"sense_key":"involvement%1:24:00::","synset_id":"involvement.n.02"},{"index":2,"sense_key":"involvement%1:04:00::","synset_id":"engagement.n.07"},{"index":3,"sense_key":"involvement%1:26:00::","synset_id":"affair.n.02"},{"index":4,"sense_key":"involvement%1:09:00::","synset_id":"interest.n.01"},{"index":5,"sense_key":"involvement%1:26:01::","synset_id":"participation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"involvement%1:04:00::","call_ids":["semeval2015.d002.s002.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"involvement%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":198,"reasoning_output_tokens":182},"cost":{"currency":"USD","total_usd":0.008225000000000001,"input_uncached_usd":0.002285,"input_cached_usd":0.0,"output_usd":0.00594,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.513590499991551}
{"item_id":"semeval2015.d002.s002.t010","gold_sense_keys":["service%1:04:08::"],"candidates":[{"index":1,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":2,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":3,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":4,"sense_key":"service%1:04:06::","synset_id":"service.n.15"},{"index":5,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":6,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":7,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":8,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":9,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":10,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":11,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":12,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":13,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":14,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":15,"sense_key":"service%1:14:05::","synset_id":"service.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"service%1:04:08::","call_ids":["semeval2015.d002.s002.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"service%1:04:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":824,"cached_input_tokens":0,"output_tokens":186,"reasoning_output_tokens":170},"cost":{"currency":"USD","total_usd":0.0097,"input_uncached_usd":0.00412,"input_cached_usd":0.0,"output_usd":0.00558,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1403473330428824}
{"item_id":"semeval2015.d002.s002.t011","gold_sense_keys":["provision%1:04:01::"],"candidates":[{"index":1,"sense_key":"provision%1:10:00::","synset_id":"provision.n.01"},{"index":2,"sense_key":"provision%1:21:00::","synset_id":"provision.n.04"},{"index":3,"sense_key":"provision%1:04:01::","synset_id":"provision.n.02"},{"index":4,"sense_key":"provision%1:09:00::","synset_id":"planning.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"provision%1:04:01::","call_ids":["semeval2015.d002.s002.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"provision%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":394,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.00341,"input_uncached_usd":0.00197,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4616397500503808}
{"item_id":"semeval2015.d002.s002.t012","gold_sense_keys":["find%2:32:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":2,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":3,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":4,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":5,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":6,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":7,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":8,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":9,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"},{"index":10,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":11,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":12,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":13,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":14,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":15,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":16,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"find%2:32:00::","call_ids":["semeval2015.d002.s002.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"find%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1036,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":111},"cost":{"currency":"USD","total_usd":0.008990000000000001,"input_uncached_usd":0.005180000000000001,"input_cached_usd":0.0,"output_usd":0.00381,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.536674875067547}
{"item_id":"semeval2015.d002.s002.t013","gold_sense_keys":["more%4:02:00::"],"candidates":[{"index":1,"sense_key":"more%4:02:00::","synset_id":"more.r.01"},{"index":2,"sense_key":"more%4:02:01::","synset_id":"more.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%4:02:00::","call_ids":["semeval2015.d002.s002.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":329,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":88},"cost":{"currency":"USD","total_usd":0.004735,"input_uncached_usd":0.0016450000000000002,"input_cached_usd":0.0,"output_usd":0.00309,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1898084999993443}
{"item_id":"semeval2015.d002.s002.t014","gold_sense_keys":["common%3:00:01::","common%5:00:00:familiar:02"],"candidates":[{"index":1,"sense_key":"common%5:00:00:lowborn:00","synset_id":"common.s.04"},{"index":2,"sense_key":"common%5:00:00:unrefined:01","synset_id":"coarse.s.01"},{"index":3,"sense_key":"common%3:00:02::","synset_id":"common.a.01"},{"index":4,"sense_key":"common%5:00:00:shared:00","synset_id":"common.s.01"},{"index":5,"sense_key":"common%5:00:00:inferior:02","synset_id":"coarse.s.02"},{"index":6,"sense_key":"common%5:00:00:familiar:02","synset_id":"common.s.02"},{"index":7,"sense_key":"common%5:00:00:informal:02","synset_id":"common.s.03"},{"index":8,"sense_key":"common%5:00:00:ordinary:00","synset_id":"common.s.07"},{"index":9,"sense_key":"common%3:00:01::","synset_id":"common.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"common%5:00:00:familiar:02","call_ids":["semeval2015.d002.s002.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"common%5:00:00:familiar:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":684,"cached_input_tokens":0,"output_tokens":4424,"reasoning_output_tokens":4360},"cost":{"currency":"USD","total_usd":0.13614,"input_uncached_usd":0.0034200000000000003,"input_cached_usd":0.0,"output_usd":0.13272,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":65.86203362501692}
{"item_id":"semeval2015.d002.s003.t002","gold_sense_keys":["variety%1:07:00::","variety%1:14:01::"],"candidates":[{"index":1,"sense_key":"variety%1:09:01::","synset_id":"kind.n.01"},{"index":2,"sense_key":"variety%1:07:01::","synset_id":"variety.n.06"},{"index":3,"sense_key":"variety%1:10:00::","synset_id":"variety_show.n.01"},{"index":4,"sense_key":"variety%1:07:00::","synset_id":"diverseness.n.01"},{"index":5,"sense_key":"variety%1:14:01::","synset_id":"assortment.n.01"},{"index":6,"sense_key":"variety%1:14:00::","synset_id":"variety.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"variety%1:07:00::","call_ids":["semeval2015.d002.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"variety%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":314,"reasoning_output_tokens":298},"cost":{"currency":"USD","total_usd":0.012345,"input_uncached_usd":0.002925,"input_cached_usd":0.0,"output_usd":0.00942,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.473712874925695}
{"item_id":"semeval2015.d002.s003.t003","gold_sense_keys":["different%3:00:02::","different%5:00:01:other:00"],"candidates":[{"index":1,"sense_key":"different%5:00:00:other:00","synset_id":"different.s.01"},{"index":2,"sense_key":"different%3:00:02::","synset_id":"unlike.a.01"},{"index":3,"sense_key":"different%5:00:01:other:00","synset_id":"different.s.03"},{"index":4,"sense_key":"different%5:00:00:unusual:00","synset_id":"different.s.02"},{"index":5,"sense_key":"different%3:00:00::","synset_id":"different.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"different%5:00:01:other:00","call_ids":["semeval2015.d002.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"different%5:00:01:other:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":211,"reasoning_output_tokens":195},"cost":{"currency":"USD","total_usd":0.008780000000000001,"input_uncached_usd":0.0024500000000000004,"input_cached_usd":0.0,"output_usd":0.0063300000000000006,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.458996374974959}
{"item_id":"semeval2015.d002.s003.t005","gold_sense_keys":["play%2:41:12::"],"candidates":[{"index":1,"sense_key":"play%2:41:00::","synset_id":"play.v.11"},{"index":2,"sense_key":"play%2:33:08::","synset_id":"bet.v.02"},{"index":3,"sense_key":"play%2:29:01::","synset_id":"act.v.05"},{"index":4,"sense_key":"play%2:36:00::","synset_id":"play.v.06"},{"index":5,"sense_key":"play%2:33:01::","synset_id":"play.v.10"},{"index":6,"sense_key":"play%2:36:06::","synset_id":"play.v.12"},{"index":7,"sense_key":"play%2:36:12::","synset_id":"play.v.03"},{"index":8,"sense_key":"play%2:42:00::","synset_id":"play.v.22"},{"index":9,"sense_key":"play%2:33:02::","synset_id":"meet.v.10"},{"index":10,"sense_key":"play%2:36:02::","synset_id":"act.v.03"},{"index":11,"sense_key":"play%2:38:03::","synset_id":"play.v.24"},{"index":12,"sense_key":"play%2:38:00::","synset_id":"play.v.09"},{"index":13,"sense_key":"play%2:36:10::","synset_id":"act.v.10"},{"index":14,"sense_key":"play%2:36:01::","synset_id":"play.v.07"},{"index":15,"sense_key":"play%2:41:12::","synset_id":"play.v.02"},{"index":16,"sense_key":"play%2:33:09::","synset_id":"play.v.31"},{"index":17,"sense_key":"play%2:33:13::","synset_id":"play.v.32"},{"index":18,"sense_key":"play%2:36:03::","synset_id":"play.v.14"},{"index":19,"sense_key":"play%2:36:05::","synset_id":"play.v.18"},{"index":20,"sense_key":"play%2:29:04::","synset_id":"play.v.35"},{"index":21,"sense_key":"play%2:29:00::","synset_id":"play.v.16"},{"index":22,"sense_key":"play%2:33:12::","synset_id":"play.v.33"},{"index":23,"sense_key":"play%2:41:03::","synset_id":"play.v.05"},{"index":24,"sense_key":"play%2:35:00::","synset_id":"toy.v.02"},{"index":25,"sense_key":"play%2:36:11::","synset_id":"play.v.13"},{"index":26,"sense_key":"play%2:31:01::","synset_id":"dally.v.04"},{"index":27,"sense_key":"play%2:35:07::","synset_id":"play.v.28"},{"index":28,"sense_key":"play%2:36:04::","synset_id":"play.v.26"},{"index":29,"sense_key":"play%2:41:02::","synset_id":"play.v.17"},{"index":30,"sense_key":"play%2:41:13::","synset_id":"dally.v.01"},{"index":31,"sense_key":"play%2:33:03::","synset_id":"play.v.15"},{"index":32,"sense_key":"play%2:34:13::","synset_id":"play.v.20"},{"index":33,"sense_key":"play%2:33:07::","synset_id":"play.v.29"},{"index":34,"sense_key":"play%2:33:00::","synset_id":"play.v.01"},{"index":35,"sense_key":"play%2:36:13::","synset_id":"bring.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"play%2:41:12::","call_ids":["semeval2015.d002.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"play%2:41:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1818,"cached_input_tokens":0,"output_tokens":216,"reasoning_output_tokens":200},"cost":{"currency":"USD","total_usd":0.01557,"input_uncached_usd":0.00909,"input_cached_usd":0.0,"output_usd":0.0064800000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.03286733396817}
{"item_id":"semeval2015.d002.s003.t006","gold_sense_keys":["public%5:00:00:common:02"],"candidates":[{"index":1,"sense_key":"public%3:00:00::","synset_id":"public.a.01"},{"index":2,"sense_key":"public%5:00:00:common:02","synset_id":"public.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"public%3:00:00::","call_ids":["semeval2015.d002.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"public%3:00:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":357,"cached_input_tokens":0,"output_tokens":226,"reasoning_output_tokens":211},"cost":{"currency":"USD","total_usd":0.008565,"input_uncached_usd":0.0017850000000000001,"input_cached_usd":0.0,"output_usd":0.0067800000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6371731250546873}
{"item_id":"semeval2015.d002.s003.t007","gold_sense_keys":["welfare%1:04:00::"],"candidates":[{"index":1,"sense_key":"welfare%1:07:00::","synset_id":"benefit.n.02"},{"index":2,"sense_key":"welfare%1:04:00::","synset_id":"social_welfare.n.01"},{"index":3,"sense_key":"welfare%1:26:00::","synset_id":"wellbeing.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"welfare%1:04:00::","call_ids":["semeval2015.d002.s003.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"welfare%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":248,"reasoning_output_tokens":232},"cost":{"currency":"USD","total_usd":0.00953,"input_uncached_usd":0.0020900000000000003,"input_cached_usd":0.0,"output_usd":0.00744,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.3741552910069}
{"item_id":"semeval2015.d002.s004.t000","gold_sense_keys":["foundation%1:14:00::"],"candidates":[{"index":1,"sense_key":"foundation%1:14:00::","synset_id":"foundation.n.02"},{"index":2,"sense_key":"foundation%1:24:00::","synset_id":"foundation.n.01"},{"index":3,"sense_key":"foundation%1:09:01::","synset_id":"foundation.n.04"},{"index":4,"sense_key":"foundation%1:06:00::","synset_id":"foundation.n.03"},{"index":5,"sense_key":"foundation%1:09:00::","synset_id":"basis.n.02"},{"index":6,"sense_key":"foundation%1:04:00::","synset_id":"initiation.n.02"},{"index":7,"sense_key":"foundation%1:06:01::","synset_id":"foundation_garment.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"foundation%1:14:00::","call_ids":["semeval2015.d002.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"foundation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":96},"cost":{"currency":"USD","total_usd":0.006515,"input_uncached_usd":0.0031850000000000003,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.293685207958333}
{"item_id":"semeval2015.d002.s004.t002","gold_sense_keys":["commission%1:14:00::","commission%1:14:01::"],"candidates":[{"index":1,"sense_key":"commission%1:04:02::","synset_id":"commission.n.03"},{"index":2,"sense_key":"commission%1:14:01::","synset_id":"deputation.n.01"},{"index":3,"sense_key":"commission%1:14:00::","synset_id":"committee.n.01"},{"index":4,"sense_key":"commission%1:26:00::","synset_id":"commission.n.04"},{"index":5,"sense_key":"commission%1:04:00::","synset_id":"mission.n.03"},{"index":6,"sense_key":"commission%1:04:01::","synset_id":"perpetration.n.01"},{"index":7,"sense_key":"commission%1:10:00::","synset_id":"commission.n.07"},{"index":8,"sense_key":"commission%1:10:01::","synset_id":"commission.n.06"},{"index":9,"sense_key":"commission%1:21:00::","synset_id":"commission.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"commission%1:14:00::","call_ids":["semeval2015.d002.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"commission%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":686,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01927,"input_uncached_usd":0.0034300000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.289653290994465}
{"item_id":"semeval2015.d002.s004.t004","gold_sense_keys":["ministry%1:14:00::"],"candidates":[{"index":1,"sense_key":"ministry%1:14:01::","synset_id":"ministry.n.01"},{"index":2,"sense_key":"ministry%1:06:00::","synset_id":"ministry.n.02"},{"index":3,"sense_key":"ministry%1:14:00::","synset_id":"ministry.n.03"},{"index":4,"sense_key":"ministry%1:04:00::","synset_id":"ministry.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ministry%1:14:00::","call_ids":["semeval2015.d002.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ministry%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":423,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.003615,"input_uncached_usd":0.002115,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3290168329840526}
{"item_id":"semeval2015.d002.s004.t005","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":2,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"},{"index":3,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"},{"index":4,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"},{"index":5,"sense_key":"social%3:00:00::","synset_id":"social.a.02"},{"index":6,"sense_key":"social%3:01:00::","synset_id":"social.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":83},"cost":{"currency":"USD","total_usd":0.005755,"input_uncached_usd":0.0028150000000000002,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.232728375005536}
{"item_id":"semeval2015.d002.s004.t006","gold_sense_keys":["affairs%1:04:00::"],"candidates":[{"index":1,"sense_key":"matter%1:09:01::","synset_id":"matter.n.01"},{"index":2,"sense_key":"affairs%1:09:00::","synset_id":"personal_business.n.01"},{"index":3,"sense_key":"affairs%1:04:00::","synset_id":"affairs.n.02"},{"index":4,"sense_key":"affair%1:11:00::","synset_id":"affair.n.03"},{"index":5,"sense_key":"affair%1:26:00::","synset_id":"affair.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"affairs%1:04:00::","call_ids":["semeval2015.d002.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"affairs%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.005025,"input_uncached_usd":0.002625,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.42334229091648}
{"item_id":"semeval2015.d002.s004.t007","gold_sense_keys":["regional%5:00:00:territorial:00"],"candidates":[{"index":1,"sense_key":"regional%3:01:00::","synset_id":"regional.a.01"},{"index":2,"sense_key":"regional%5:00:00:territorial:00","synset_id":"regional.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"regional%5:00:00:territorial:00","call_ids":["semeval2015.d002.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"regional%5:00:00:territorial:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":372,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.00468,"input_uncached_usd":0.00186,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.48502370796632}
{"item_id":"semeval2015.d002.s004.t008","gold_sense_keys":["government%1:14:00::"],"candidates":[{"index":1,"sense_key":"government%1:09:00::","synset_id":"government.n.03"},{"index":2,"sense_key":"government%1:09:01::","synset_id":"politics.n.02"},{"index":3,"sense_key":"government%1:14:00::","synset_id":"government.n.01"},{"index":4,"sense_key":"government%1:04:00::","synset_id":"government.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"government%1:14:00::","call_ids":["semeval2015.d002.s004.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"government%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.004350000000000001,"input_uncached_usd":0.0024600000000000004,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0248547500232235}
{"item_id":"semeval2015.d002.s004.t010","gold_sense_keys":["conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":2,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"},{"index":3,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"conference%1:14:00::","call_ids":["semeval2015.d002.s004.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.0050100000000000006,"input_uncached_usd":0.0020700000000000002,"input_cached_usd":0.0,"output_usd":0.00294,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6957732080481946}
{"item_id":"semeval2015.d002.s004.t011","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"},{"index":2,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"},{"index":3,"sense_key":"social%3:00:00::","synset_id":"social.a.02"},{"index":4,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":5,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"},{"index":6,"sense_key":"social%3:01:00::","synset_id":"social.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s004.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.005215000000000001,"input_uncached_usd":0.0028150000000000002,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3121036250377074}
{"item_id":"semeval2015.d002.s004.t012","gold_sense_keys":["exclusion%1:26:00::"],"candidates":[{"index":1,"sense_key":"exclusion%1:09:00::","synset_id":"exception.n.01"},{"index":2,"sense_key":"exclusion%1:26:00::","synset_id":"exclusion.n.01"},{"index":3,"sense_key":"exclusion%1:26:01::","synset_id":"excommunication.n.01"},{"index":4,"sense_key":"exclusion%1:04:01::","synset_id":"ejection.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"exclusion%1:26:00::","call_ids":["semeval2015.d002.s004.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"exclusion%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":85},"cost":{"currency":"USD","total_usd":0.005335,"input_uncached_usd":0.0023350000000000003,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5558257090160623}
{"item_id":"semeval2015.d002.s004.t013","gold_sense_keys":["major%3:00:02::","major%3:00:06::"],"candidates":[{"index":1,"sense_key":"major%3:00:03::","synset_id":"major.a.07"},{"index":2,"sense_key":"major%3:00:02::","synset_id":"major.a.02"},{"index":3,"sense_key":"major%3:00:06::","synset_id":"major.a.01"},{"index":4,"sense_key":"major%3:00:01::","synset_id":"major.a.03"},{"index":5,"sense_key":"major%3:00:04::","synset_id":"major.a.05"},{"index":6,"sense_key":"major%5:00:00:senior:00","synset_id":"major.s.01"},{"index":7,"sense_key":"major%3:00:05::","synset_id":"major.a.04"},{"index":8,"sense_key":"major%3:00:07::","synset_id":"major.a.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"major%3:00:07::","call_ids":["semeval2015.d002.s004.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"major%3:00:07::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":575,"cached_input_tokens":0,"output_tokens":2082,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.065335,"input_uncached_usd":0.0028750000000000004,"input_cached_usd":0.0,"output_usd":0.06246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":35.94745391700417}
{"item_id":"semeval2015.d002.s004.t014","gold_sense_keys":["challenge%1:26:00::"],"candidates":[{"index":1,"sense_key":"challenge%1:10:03::","synset_id":"challenge.n.05"},{"index":2,"sense_key":"challenge%1:10:00::","synset_id":"challenge.n.02"},{"index":3,"sense_key":"challenge%1:10:02::","synset_id":"challenge.n.04"},{"index":4,"sense_key":"challenge%1:26:00::","synset_id":"challenge.n.01"},{"index":5,"sense_key":"challenge%1:10:01::","synset_id":"challenge.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"challenge%1:26:00::","call_ids":["semeval2015.d002.s004.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"challenge%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0042250000000000005,"input_uncached_usd":0.0023350000000000003,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.300234750029631}
{"item_id":"semeval2015.d002.s004.t015","gold_sense_keys":["public%5:00:00:common:02"],"candidates":[{"index":1,"sense_key":"public%3:00:00::","synset_id":"public.a.01"},{"index":2,"sense_key":"public%5:00:00:common:02","synset_id":"public.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"public%3:00:00::","call_ids":["semeval2015.d002.s004.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"public%3:00:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":388,"cached_input_tokens":0,"output_tokens":292,"reasoning_output_tokens":277},"cost":{"currency":"USD","total_usd":0.010700000000000001,"input_uncached_usd":0.00194,"input_cached_usd":0.0,"output_usd":0.00876,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.120967749971896}
{"item_id":"semeval2015.d002.s004.t016","gold_sense_keys":["welfare%1:04:00::"],"candidates":[{"index":1,"sense_key":"welfare%1:07:00::","synset_id":"benefit.n.02"},{"index":2,"sense_key":"welfare%1:26:00::","synset_id":"wellbeing.n.01"},{"index":3,"sense_key":"welfare%1:04:00::","synset_id":"social_welfare.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"welfare%1:04:00::","call_ids":["semeval2015.d002.s004.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"welfare%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":138},"cost":{"currency":"USD","total_usd":0.006865000000000001,"input_uncached_usd":0.002245,"input_cached_usd":0.0,"output_usd":0.00462,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.875915916985832}
{"item_id":"semeval2015.d002.s005.t000","gold_sense_keys":["foundation%1:14:00::"],"candidates":[{"index":1,"sense_key":"foundation%1:06:01::","synset_id":"foundation_garment.n.01"},{"index":2,"sense_key":"foundation%1:09:00::","synset_id":"basis.n.02"},{"index":3,"sense_key":"foundation%1:14:00::","synset_id":"foundation.n.02"},{"index":4,"sense_key":"foundation%1:09:01::","synset_id":"foundation.n.04"},{"index":5,"sense_key":"foundation%1:24:00::","synset_id":"foundation.n.01"},{"index":6,"sense_key":"foundation%1:06:00::","synset_id":"foundation.n.03"},{"index":7,"sense_key":"foundation%1:04:00::","synset_id":"initiation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"foundation%1:14:00::","call_ids":["semeval2015.d002.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"foundation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.005875,"input_uncached_usd":0.0033250000000000003,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1611019159900025}
{"item_id":"semeval2015.d002.s005.t002","gold_sense_keys":["publish%2:32:00::","publish%2:36:01::"],"candidates":[{"index":1,"sense_key":"publish%2:32:00::","synset_id":"publish.v.02"},{"index":2,"sense_key":"publish%2:36:01::","synset_id":"print.v.01"},{"index":3,"sense_key":"publish%2:36:00::","synset_id":"publish.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"publish%2:32:00::","call_ids":["semeval2015.d002.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"publish%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":173,"reasoning_output_tokens":158},"cost":{"currency":"USD","total_usd":0.0076100000000000004,"input_uncached_usd":0.0024200000000000003,"input_cached_usd":0.0,"output_usd":0.00519,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9991403749445453}
{"item_id":"semeval2015.d002.s005.t003","gold_sense_keys":["comparative%3:01:00::"],"candidates":[{"index":1,"sense_key":"comparative%3:00:00::","synset_id":"relative.a.01"},{"index":2,"sense_key":"comparative%3:01:00::","synset_id":"comparative.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"comparative%3:01:00::","call_ids":["semeval2015.d002.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"comparative%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.00391,"input_uncached_usd":0.00205,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8354108330095187}
{"item_id":"semeval2015.d002.s005.t004","gold_sense_keys":["study%1:04:00::","study%1:10:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":3,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":4,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":5,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":6,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":7,"sense_key":"study%1:10:01::","synset_id":"study.n.10"},{"index":8,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":9,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":10,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d002.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.0073100000000000005,"input_uncached_usd":0.0037700000000000003,"input_cached_usd":0.0,"output_usd":0.00354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.239263292052783}
{"item_id":"semeval2015.d002.s005.t005","gold_sense_keys":["part%1:09:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":4,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":5,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":6,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":7,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":8,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":9,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"},{"index":10,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":11,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":12,"sense_key":"part%1:24:00::","synset_id":"part.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"part%1:24:00::","call_ids":["semeval2015.d002.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"part%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":933,"cached_input_tokens":0,"output_tokens":388,"reasoning_output_tokens":372},"cost":{"currency":"USD","total_usd":0.016305,"input_uncached_usd":0.004665000000000001,"input_cached_usd":0.0,"output_usd":0.011640000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.246116208960302}
{"item_id":"semeval2015.d002.s005.t007","gold_sense_keys":["assess%2:31:00::"],"candidates":[{"index":1,"sense_key":"assess%2:40:01::","synset_id":"assess.v.02"},{"index":2,"sense_key":"assess%2:31:01::","synset_id":"assess.v.04"},{"index":3,"sense_key":"assess%2:31:00::","synset_id":"measure.v.04"},{"index":4,"sense_key":"assess%2:40:00::","synset_id":"tax.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"assess%2:31:00::","call_ids":["semeval2015.d002.s005.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"assess%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.0046700000000000005,"input_uncached_usd":0.0026000000000000003,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.68042383401189}
{"item_id":"semeval2015.d002.s005.t008","gold_sense_keys":["extent%1:26:00::"],"candidates":[{"index":1,"sense_key":"extent%1:07:00::","synset_id":"extent.n.02"},{"index":2,"sense_key":"extent%1:26:00::","synset_id":"extent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"extent%1:26:00::","call_ids":["semeval2015.d002.s005.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"extent%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.0039700000000000004,"input_uncached_usd":0.0021100000000000003,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.004800167051144}
{"item_id":"semeval2015.d002.s005.t009","gold_sense_keys":["direct%5:00:00:immediate:00"],"candidates":[{"index":1,"sense_key":"direct%3:00:03::","synset_id":"direct.a.06"},{"index":2,"sense_key":"direct%5:00:00:primary:00","synset_id":"direct.s.02"},{"index":3,"sense_key":"direct%5:00:00:absolute:00","synset_id":"direct.s.04"},{"index":4,"sense_key":"direct%5:00:00:immediate:00","synset_id":"direct.s.01"},{"index":5,"sense_key":"direct%3:00:01::","synset_id":"direct.a.07"},{"index":6,"sense_key":"direct%3:00:00::","synset_id":"direct.a.01"},{"index":7,"sense_key":"direct%3:00:02::","synset_id":"direct.a.03"},{"index":8,"sense_key":"direct%5:00:00:exact:00","synset_id":"direct.s.03"},{"index":9,"sense_key":"direct%3:00:05::","synset_id":"direct.a.05"},{"index":10,"sense_key":"direct%3:00:04::","synset_id":"lineal.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"direct%5:00:00:immediate:00","call_ids":["semeval2015.d002.s005.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"direct%5:00:00:immediate:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":744,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":91},"cost":{"currency":"USD","total_usd":0.00693,"input_uncached_usd":0.00372,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4190811249427497}
{"item_id":"semeval2015.d002.s005.t010","gold_sense_keys":["participation%1:04:00::"],"candidates":[{"index":1,"sense_key":"participation%1:04:00::","synset_id":"engagement.n.07"},{"index":2,"sense_key":"participation%1:26:00::","synset_id":"participation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"participation%1:04:00::","call_ids":["semeval2015.d002.s005.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"participation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":80},"cost":{"currency":"USD","total_usd":0.0050100000000000006,"input_uncached_usd":0.00216,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9022559999721125}
{"item_id":"semeval2015.d002.s005.t012","gold_sense_keys":["work%1:04:01::"],"candidates":[{"index":1,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":2,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":3,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"},{"index":4,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":5,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":6,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":7,"sense_key":"work%1:19:00::","synset_id":"work.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"work%1:04:00::","call_ids":["semeval2015.d002.s005.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"work%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019345,"input_uncached_usd":0.0035050000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.99007891700603}
{"item_id":"semeval2015.d002.s005.t013","gold_sense_keys":["increase%2:30:02::"],"candidates":[{"index":1,"sense_key":"increase%2:30:00::","synset_id":"increase.v.01"},{"index":2,"sense_key":"increase%2:30:02::","synset_id":"increase.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"increase%2:30:02::","call_ids":["semeval2015.d002.s005.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"increase%2:30:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":413,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":89},"cost":{"currency":"USD","total_usd":0.005185,"input_uncached_usd":0.002065,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2564538340084255}
{"item_id":"semeval2015.d002.s006.t000","gold_sense_keys":["study%1:10:00::"],"candidates":[{"index":1,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":2,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":3,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":4,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":5,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":6,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":7,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":8,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":9,"sense_key":"study%1:10:01::","synset_id":"study.n.10"},{"index":10,"sense_key":"study%1:09:03::","synset_id":"study.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d002.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":207,"reasoning_output_tokens":191},"cost":{"currency":"USD","total_usd":0.009975000000000001,"input_uncached_usd":0.003765,"input_cached_usd":0.0,"output_usd":0.00621,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0917532079620287}
{"item_id":"semeval2015.d002.s006.t001","gold_sense_keys":["base%2:31:00::"],"candidates":[{"index":1,"sense_key":"base%2:31:00::","synset_id":"establish.v.08"},{"index":2,"sense_key":"base%2:42:04::","synset_id":"base.v.02"},{"index":3,"sense_key":"base%2:34:00::","synset_id":"free-base.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"base%2:31:00::","call_ids":["semeval2015.d002.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"base%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.004070000000000001,"input_uncached_usd":0.0022700000000000003,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8803492499282584}
{"item_id":"semeval2015.d002.s006.t002","gold_sense_keys":["personal%3:00:00::","personal%5:00:00:subjective:00"],"candidates":[{"index":1,"sense_key":"personal%3:01:00::","synset_id":"personal.a.05"},{"index":2,"sense_key":"personal%3:01:01::","synset_id":"personal.a.03"},{"index":3,"sense_key":"personal%5:00:00:physical:00","synset_id":"personal.s.02"},{"index":4,"sense_key":"personal%3:00:00::","synset_id":"personal.a.01"},{"index":5,"sense_key":"personal%5:00:00:subjective:00","synset_id":"personal.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"personal%3:00:00::","call_ids":["semeval2015.d002.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"personal%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01832,"input_uncached_usd":0.00248,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.31305674999021}
{"item_id":"semeval2015.d002.s006.t003","gold_sense_keys":["interview%1:10:01::"],"candidates":[{"index":1,"sense_key":"interview%1:10:00::","synset_id":"consultation.n.01"},{"index":2,"sense_key":"interview%1:10:01::","synset_id":"interview.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"interview%1:10:01::","call_ids":["semeval2015.d002.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"interview%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.004030000000000001,"input_uncached_usd":0.0022,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6900235830107704}
{"item_id":"semeval2015.d002.s006.t004","gold_sense_keys":["senior%3:00:00::"],"candidates":[{"index":1,"sense_key":"senior%5:00:00:old:02","synset_id":"aged.s.01"},{"index":2,"sense_key":"senior%3:00:00::","synset_id":"senior.a.01"},{"index":3,"sense_key":"senior%5:00:00:last:00","synset_id":"senior.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"senior%3:00:00::","call_ids":["semeval2015.d002.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"senior%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00423,"input_uncached_usd":0.0024300000000000003,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6934894589940086}
{"item_id":"semeval2015.d002.s006.t005","gold_sense_keys":["official%1:18:01::"],"candidates":[{"index":1,"sense_key":"official%1:18:01::","synset_id":"official.n.01"},{"index":2,"sense_key":"official%1:18:00::","synset_id":"official.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"official%1:18:01::","call_ids":["semeval2015.d002.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"official%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":413,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.003955,"input_uncached_usd":0.002065,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1269275000086054}
{"item_id":"semeval2015.d002.s006.t006","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"},{"index":2,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"},{"index":3,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":4,"sense_key":"social%3:01:00::","synset_id":"social.a.01"},{"index":5,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"},{"index":6,"sense_key":"social%3:00:00::","synset_id":"social.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s006.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":590,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":127},"cost":{"currency":"USD","total_usd":0.00721,"input_uncached_usd":0.0029500000000000004,"input_cached_usd":0.0,"output_usd":0.00426,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.240317916031927}
{"item_id":"semeval2015.d002.s006.t007","gold_sense_keys":["representative%1:18:00::","representative%1:18:02::"],"candidates":[{"index":1,"sense_key":"representative%1:09:00::","synset_id":"example.n.01"},{"index":2,"sense_key":"representative%1:18:02::","synset_id":"spokesperson.n.01"},{"index":3,"sense_key":"representative%1:18:00::","synset_id":"representative.n.01"},{"index":4,"sense_key":"representative%1:18:01::","synset_id":"congressman.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"representative%1:18:00::","call_ids":["semeval2015.d002.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"representative%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":220,"reasoning_output_tokens":204},"cost":{"currency":"USD","total_usd":0.009145,"input_uncached_usd":0.0025450000000000004,"input_cached_usd":0.0,"output_usd":0.0066,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.04444766708184}
{"item_id":"semeval2015.d002.s006.t008","gold_sense_keys":["national%3:00:01::","national%3:01:00::"],"candidates":[{"index":1,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":2,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":3,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":4,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":5,"sense_key":"national%3:01:01::","synset_id":"national.a.07"},{"index":6,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":7,"sense_key":"national%3:01:00::","synset_id":"national.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"national%3:00:01::","call_ids":["semeval2015.d002.s006.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":147},"cost":{"currency":"USD","total_usd":0.00792,"input_uncached_usd":0.00303,"input_cached_usd":0.0,"output_usd":0.00489,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.5749917910434306}
{"item_id":"semeval2015.d002.s006.t009","gold_sense_keys":["level%1:26:01::"],"candidates":[{"index":1,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"},{"index":2,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":3,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":4,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":5,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":6,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:00::","synset_id":"level.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"level%1:26:01::","call_ids":["semeval2015.d002.s006.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"level%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.019345,"input_uncached_usd":0.0035050000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.957360332948156}
{"item_id":"semeval2015.d002.s006.t010","gold_sense_keys":["organisation%1:14:00::","organisation%1:14:01::"],"candidates":[{"index":1,"sense_key":"organisation%1:14:00::","synset_id":"organization.n.01"},{"index":2,"sense_key":"organisation%1:04:02::","synset_id":"constitution.n.02"},{"index":3,"sense_key":"organisation%1:04:00::","synset_id":"organization.n.04"},{"index":4,"sense_key":"organisation%1:04:01::","synset_id":"organization.n.06"},{"index":5,"sense_key":"organisation%1:09:00::","synset_id":"arrangement.n.03"},{"index":6,"sense_key":"organisation%1:14:01::","synset_id":"administration.n.02"},{"index":7,"sense_key":"organisation%1:07:00::","synset_id":"organization.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"organisation%1:14:00::","call_ids":["semeval2015.d002.s006.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"organisation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":716,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.00631,"input_uncached_usd":0.0035800000000000003,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.771480750059709}
{"item_id":"semeval2015.d002.s006.t013","gold_sense_keys":["country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":4,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":5,"sense_key":"country%1:14:00::","synset_id":"state.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"country%1:15:00::","call_ids":["semeval2015.d002.s006.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"country%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":113},"cost":{"currency":"USD","total_usd":0.006775,"input_uncached_usd":0.002935,"input_cached_usd":0.0,"output_usd":0.00384,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3937683750409633}
{"item_id":"semeval2015.d002.s007.t000","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":2,"sense_key":"important%3:00:00::","synset_id":"important.a.01"},{"index":3,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"},{"index":4,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"},{"index":5,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"important%3:00:00::","call_ids":["semeval2015.d002.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"important%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.007300000000000001,"input_uncached_usd":0.0029800000000000004,"input_cached_usd":0.0,"output_usd":0.00432,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7408576669404283}
{"item_id":"semeval2015.d002.s007.t001","gold_sense_keys":["industrial%3:01:00::"],"candidates":[{"index":1,"sense_key":"industrial%3:01:00::","synset_id":"industrial.a.01"},{"index":2,"sense_key":"industrial%5:00:00:blue-collar:00","synset_id":"industrial.s.01"},{"index":3,"sense_key":"industrial%3:00:00::","synset_id":"industrial.a.02"},{"index":4,"sense_key":"industrial%5:00:00:heavy-duty:00","synset_id":"industrial.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"industrial%3:01:00::","call_ids":["semeval2015.d002.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"industrial%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.00496,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.58897329098545}
{"item_id":"semeval2015.d002.s007.t002","gold_sense_keys":["sector%1:14:00::"],"candidates":[{"index":1,"sense_key":"sector%1:15:00::","synset_id":"sector.n.05"},{"index":2,"sense_key":"sector%1:06:00::","synset_id":"sector.n.06"},{"index":3,"sense_key":"sector%1:23:00::","synset_id":"sector.n.04"},{"index":4,"sense_key":"sector%1:09:00::","synset_id":"sector.n.03"},{"index":5,"sense_key":"sector%1:25:00::","synset_id":"sector.n.01"},{"index":6,"sense_key":"sector%1:14:00::","synset_id":"sector.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"sector%1:14:00::","call_ids":["semeval2015.d002.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"sector%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":88},"cost":{"currency":"USD","total_usd":0.00596,"input_uncached_usd":0.00284,"input_cached_usd":0.0,"output_usd":0.00312,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1185698750196025}
{"item_id":"semeval2015.d002.s007.t003","gold_sense_keys":["metal%1:27:00::"],"candidates":[{"index":1,"sense_key":"metal%1:27:01::","synset_id":"alloy.n.01"},{"index":2,"sense_key":"metal%1:27:00::","synset_id":"metallic_element.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"metal%1:27:00::","call_ids":["semeval2015.d002.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"metal%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":132},"cost":{"currency":"USD","total_usd":0.0068200000000000005,"input_uncached_usd":0.00238,"input_cached_usd":0.0,"output_usd":0.00444,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1203429580200464}
{"item_id":"semeval2015.d002.s007.t004","gold_sense_keys":["industry%1:14:00::"],"candidates":[{"index":1,"sense_key":"industry%1:04:00::","synset_id":"industry.n.02"},{"index":2,"sense_key":"industry%1:14:00::","synset_id":"industry.n.01"},{"index":3,"sense_key":"industry%1:07:00::","synset_id":"diligence.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"industry%1:14:00::","call_ids":["semeval2015.d002.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"industry%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.00508,"input_uncached_usd":0.00256,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1009884160012007}
{"item_id":"semeval2015.d002.s007.t005","gold_sense_keys":["banking%1:04:01::"],"candidates":[{"index":1,"sense_key":"banking%1:04:00::","synset_id":"banking.n.02"},{"index":2,"sense_key":"banking%1:04:01::","synset_id":"banking.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"banking%1:04:01::","call_ids":["semeval2015.d002.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"banking%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.004125,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.861831916961819}
{"item_id":"semeval2015.d002.s007.t006","gold_sense_keys":["sector%1:14:00::"],"candidates":[{"index":1,"sense_key":"sector%1:09:00::","synset_id":"sector.n.03"},{"index":2,"sense_key":"sector%1:15:00::","synset_id":"sector.n.05"},{"index":3,"sense_key":"sector%1:25:00::","synset_id":"sector.n.01"},{"index":4,"sense_key":"sector%1:14:00::","synset_id":"sector.n.02"},{"index":5,"sense_key":"sector%1:06:00::","synset_id":"sector.n.06"},{"index":6,"sense_key":"sector%1:23:00::","synset_id":"sector.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"sector%1:14:00::","call_ids":["semeval2015.d002.s007.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"sector%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.0053,"input_uncached_usd":0.00284,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.142505209078081}
{"item_id":"semeval2015.d002.s007.t008","gold_sense_keys":["include%2:30:00::","include%2:31:00::"],"candidates":[{"index":1,"sense_key":"include%2:31:00::","synset_id":"include.v.02"},{"index":2,"sense_key":"include%2:30:00::","synset_id":"include.v.03"},{"index":3,"sense_key":"include%2:41:03::","synset_id":"admit.v.03"},{"index":4,"sense_key":"include%2:42:00::","synset_id":"include.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"include%2:30:00::","call_ids":["semeval2015.d002.s007.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"include%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.034159999999999996,"input_uncached_usd":0.0027800000000000004,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":15.869168290984817}
{"item_id":"semeval2015.d002.s008.t000","gold_sense_keys":["speaker%1:18:00::"],"candidates":[{"index":1,"sense_key":"speaker%1:06:00::","synset_id":"loudspeaker.n.01"},{"index":2,"sense_key":"speaker%1:18:01::","synset_id":"speaker.n.03"},{"index":3,"sense_key":"speaker%1:18:00::","synset_id":"speaker.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"speaker%1:18:00::","call_ids":["semeval2015.d002.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"speaker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.0048850000000000005,"input_uncached_usd":0.002755,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.525493541965261}
{"item_id":"semeval2015.d002.s008.t001","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"},{"index":2,"sense_key":"social%3:00:00::","synset_id":"social.a.02"},{"index":3,"sense_key":"social%3:01:00::","synset_id":"social.a.01"},{"index":4,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"},{"index":5,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":6,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":82},"cost":{"currency":"USD","total_usd":0.006095,"input_uncached_usd":0.0031850000000000003,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3941637909738347}
{"item_id":"semeval2015.d002.s008.t002","gold_sense_keys":["exclusion%1:26:00::"],"candidates":[{"index":1,"sense_key":"exclusion%1:26:00::","synset_id":"exclusion.n.01"},{"index":2,"sense_key":"exclusion%1:09:00::","synset_id":"exception.n.01"},{"index":3,"sense_key":"exclusion%1:26:01::","synset_id":"excommunication.n.01"},{"index":4,"sense_key":"exclusion%1:04:01::","synset_id":"ejection.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"exclusion%1:26:00::","call_ids":["semeval2015.d002.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"exclusion%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":542,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":123},"cost":{"currency":"USD","total_usd":0.006880000000000001,"input_uncached_usd":0.00271,"input_cached_usd":0.0,"output_usd":0.00417,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8574499160749838}
{"item_id":"semeval2015.d002.s008.t003","gold_sense_keys":["conference%1:10:00::","conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":2,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"},{"index":3,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"conference%1:14:00::","call_ids":["semeval2015.d002.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.00406,"input_uncached_usd":0.0024400000000000003,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4995925419498235}
{"item_id":"semeval2015.d002.s008.t004","gold_sense_keys":["left%1:15:00::"],"candidates":[{"index":1,"sense_key":"left%1:08:00::","synset_id":"left.n.03"},{"index":2,"sense_key":"left%1:04:00::","synset_id":"left.n.05"},{"index":3,"sense_key":"left%1:06:00::","synset_id":"left_field.n.01"},{"index":4,"sense_key":"left%1:14:00::","synset_id":"left.n.02"},{"index":5,"sense_key":"left%1:15:00::","synset_id":"left.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"left%1:15:00::","call_ids":["semeval2015.d002.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"left%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":603,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.005115000000000001,"input_uncached_usd":0.0030150000000000003,"input_cached_usd":0.0,"output_usd":0.0021,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.28071204200387}
{"item_id":"semeval2015.d002.s008.t005","gold_sense_keys":["right%1:15:00::"],"candidates":[{"index":1,"sense_key":"right%1:06:00::","synset_id":"right_field.n.01"},{"index":2,"sense_key":"right%1:07:00::","synset_id":"right.n.01"},{"index":3,"sense_key":"right%1:07:01::","synset_id":"right.n.07"},{"index":4,"sense_key":"right%1:04:00::","synset_id":"right.n.06"},{"index":5,"sense_key":"right%1:15:00::","synset_id":"right.n.02"},{"index":6,"sense_key":"right%1:08:00::","synset_id":"right.n.05"},{"index":7,"sense_key":"right%1:14:00::","synset_id":"right.n.04"},{"index":8,"sense_key":"right%1:21:00::","synset_id":"right.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"right%1:15:00::","call_ids":["semeval2015.d002.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"right%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":775,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":80},"cost":{"currency":"USD","total_usd":0.006725,"input_uncached_usd":0.0038750000000000004,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.597972000017762}
{"item_id":"semeval2015.d002.s008.t006","gold_sense_keys":["employment%1:04:00::","employment%1:04:02::","employment%1:26:00::"],"candidates":[{"index":1,"sense_key":"employment%1:04:02::","synset_id":"employment.n.03"},{"index":2,"sense_key":"employment%1:04:01::","synset_id":"use.n.01"},{"index":3,"sense_key":"employment%1:04:00::","synset_id":"employment.n.02"},{"index":4,"sense_key":"employment%1:26:00::","synset_id":"employment.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"employment%1:26:00::","call_ids":["semeval2015.d002.s008.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"employment%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018635,"input_uncached_usd":0.002795,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.629508166923188}
{"item_id":"semeval2015.d002.s008.t007","gold_sense_keys":["industrial%3:01:00::"],"candidates":[{"index":1,"sense_key":"industrial%5:00:00:heavy-duty:00","synset_id":"industrial.s.02"},{"index":2,"sense_key":"industrial%3:00:00::","synset_id":"industrial.a.02"},{"index":3,"sense_key":"industrial%5:00:00:blue-collar:00","synset_id":"industrial.s.01"},{"index":4,"sense_key":"industrial%3:01:00::","synset_id":"industrial.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"industrial%3:01:00::","call_ids":["semeval2015.d002.s008.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"industrial%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":181,"reasoning_output_tokens":165},"cost":{"currency":"USD","total_usd":0.008005000000000002,"input_uncached_usd":0.002575,"input_cached_usd":0.0,"output_usd":0.00543,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.6822863339912146}
{"item_id":"semeval2015.d002.s008.t012","gold_sense_keys":["commission%1:14:00::","commission%1:14:01::"],"candidates":[{"index":1,"sense_key":"commission%1:21:00::","synset_id":"commission.n.02"},{"index":2,"sense_key":"commission%1:04:02::","synset_id":"commission.n.03"},{"index":3,"sense_key":"commission%1:26:00::","synset_id":"commission.n.04"},{"index":4,"sense_key":"commission%1:14:00::","synset_id":"committee.n.01"},{"index":5,"sense_key":"commission%1:14:01::","synset_id":"deputation.n.01"},{"index":6,"sense_key":"commission%1:10:00::","synset_id":"commission.n.07"},{"index":7,"sense_key":"commission%1:04:00::","synset_id":"mission.n.03"},{"index":8,"sense_key":"commission%1:04:01::","synset_id":"perpetration.n.01"},{"index":9,"sense_key":"commission%1:10:01::","synset_id":"commission.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"commission%1:14:00::","call_ids":["semeval2015.d002.s008.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"commission%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":262,"reasoning_output_tokens":247},"cost":{"currency":"USD","total_usd":0.011665000000000002,"input_uncached_usd":0.0038050000000000002,"input_cached_usd":0.0,"output_usd":0.00786,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.359905792050995}
{"item_id":"semeval2015.d002.s008.t013","gold_sense_keys":["high%3:00:02::"],"candidates":[{"index":1,"sense_key":"high%5:00:00:malodorous:00","synset_id":"gamey.s.02"},{"index":2,"sense_key":"high%5:00:00:intoxicated:00","synset_id":"high.s.04"},{"index":3,"sense_key":"high%5:00:00:elated:00","synset_id":"high.s.02"},{"index":4,"sense_key":"high%3:00:01::","synset_id":"high.a.02"},{"index":5,"sense_key":"high%3:00:02::","synset_id":"high.a.01"},{"index":6,"sense_key":"high%5:00:00:superior:01","synset_id":"eminent.s.01"},{"index":7,"sense_key":"high%3:00:03::","synset_id":"high.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"high%3:00:02::","call_ids":["semeval2015.d002.s008.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"high%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":664,"cached_input_tokens":0,"output_tokens":1302,"reasoning_output_tokens":1274},"cost":{"currency":"USD","total_usd":0.04238000000000001,"input_uncached_usd":0.0033200000000000005,"input_cached_usd":0.0,"output_usd":0.039060000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":22.951123042032123}
{"item_id":"semeval2015.d002.s008.t014","gold_sense_keys":["committee%1:14:00::"],"candidates":[{"index":1,"sense_key":"committee%1:14:00::","synset_id":"committee.n.01"},{"index":2,"sense_key":"committee%1:14:01::","synset_id":"committee.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"committee%1:14:00::","call_ids":["semeval2015.d002.s008.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"committee%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":173,"reasoning_output_tokens":157},"cost":{"currency":"USD","total_usd":0.00751,"input_uncached_usd":0.00232,"input_cached_usd":0.0,"output_usd":0.00519,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.07815433293581}
{"item_id":"semeval2015.d002.s008.t015","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":2,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":3,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":4,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":5,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"member%1:18:00::","call_ids":["semeval2015.d002.s008.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"member%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.005795,"input_uncached_usd":0.0031850000000000003,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.242900083074346}
{"item_id":"semeval2015.d002.s008.t017","gold_sense_keys":["parliament%1:14:00::"],"candidates":[{"index":1,"sense_key":"parliament%1:04:00::","synset_id":"fantan.n.02"},{"index":2,"sense_key":"parliament%1:14:00::","synset_id":"parliament.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"parliament%1:14:00::","call_ids":["semeval2015.d002.s008.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"parliament%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.004,"input_uncached_usd":0.00235,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3553187500219792}
{"item_id":"semeval2015.d002.s008.t020","gold_sense_keys":["parliament%1:14:00::"],"candidates":[{"index":1,"sense_key":"parliament%1:04:00::","synset_id":"fantan.n.02"},{"index":2,"sense_key":"parliament%1:14:00::","synset_id":"parliament.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"parliament%1:14:00::","call_ids":["semeval2015.d002.s008.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"parliament%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":33,"reasoning_output_tokens":18},"cost":{"currency":"USD","total_usd":0.00334,"input_uncached_usd":0.00235,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5342400419758633}
{"item_id":"semeval2015.d002.s008.t021","gold_sense_keys":["economic%3:01:01::"],"candidates":[{"index":1,"sense_key":"economic%3:01:00::","synset_id":"economic.a.02"},{"index":2,"sense_key":"economic%5:00:00:efficient:00","synset_id":"economic.s.01"},{"index":3,"sense_key":"economic%3:01:01::","synset_id":"economic.a.01"},{"index":4,"sense_key":"economic%5:00:00:profitable:00","synset_id":"economic.s.03"},{"index":5,"sense_key":"economic%5:00:00:worldly:00","synset_id":"economic.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"economic%3:01:01::","call_ids":["semeval2015.d002.s008.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"economic%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.005855,"input_uncached_usd":0.003125,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9732961669797078}
{"item_id":"semeval2015.d002.s008.t022","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"},{"index":2,"sense_key":"social%3:01:00::","synset_id":"social.a.01"},{"index":3,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"},{"index":4,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":5,"sense_key":"social%3:00:00::","synset_id":"social.a.02"},{"index":6,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s008.t022__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.0053750000000000004,"input_uncached_usd":0.0031850000000000003,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8759085410274565}
{"item_id":"semeval2015.d002.s008.t023","gold_sense_keys":["committee%1:14:00::"],"candidates":[{"index":1,"sense_key":"committee%1:14:00::","synset_id":"committee.n.01"},{"index":2,"sense_key":"committee%1:14:01::","synset_id":"committee.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"committee%1:14:00::","call_ids":["semeval2015.d002.s008.t023__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"committee%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":463,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":98},"cost":{"currency":"USD","total_usd":0.005705,"input_uncached_usd":0.0023150000000000002,"input_cached_usd":0.0,"output_usd":0.0033900000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1190762920305133}
{"item_id":"semeval2015.d002.s009.t000","gold_sense_keys":["case_study%1:10:00::"],"candidates":[{"index":1,"sense_key":"case_study%1:10:00::","synset_id":"case_study.n.01"},{"index":2,"sense_key":"case_study%1:04:00::","synset_id":"case_study.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case_study%1:10:00::","call_ids":["semeval2015.d002.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case_study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":338,"reasoning_output_tokens":322},"cost":{"currency":"USD","total_usd":0.01252,"input_uncached_usd":0.00238,"input_cached_usd":0.0,"output_usd":0.01014,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.768416375038214}
{"item_id":"semeval2015.d002.s009.t001","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":2,"sense_key":"national%3:01:01::","synset_id":"national.a.07"},{"index":3,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":6,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":7,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"national%3:00:01::","call_ids":["semeval2015.d002.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":657,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.034665,"input_uncached_usd":0.003285,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":17.25025545794051}
{"item_id":"semeval2015.d002.s009.t002","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"},{"index":3,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":4,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":5,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:10:00::","synset_id":"report.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2015.d002.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":715,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.005405,"input_uncached_usd":0.0035750000000000005,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.443251624936238}
{"item_id":"semeval2015.d002.s009.t003","gold_sense_keys":["contain%2:42:00::"],"candidates":[{"index":1,"sense_key":"contain%2:42:06::","synset_id":"contain.v.04"},{"index":2,"sense_key":"contain%2:41:00::","synset_id":"control.v.02"},{"index":3,"sense_key":"contain%2:42:14::","synset_id":"contain.v.05"},{"index":4,"sense_key":"contain%2:33:00::","synset_id":"check.v.18"},{"index":5,"sense_key":"contain%2:42:00::","synset_id":"incorporate.v.02"},{"index":6,"sense_key":"contain%2:42:13::","synset_id":"hold.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"contain%2:42:00::","call_ids":["semeval2015.d002.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"contain%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.00625,"input_uncached_usd":0.0034600000000000004,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.275474749971181}
{"item_id":"semeval2015.d002.s009.t006","gold_sense_keys":["present%2:32:00::"],"candidates":[{"index":1,"sense_key":"present%2:39:00::","synset_id":"show.v.01"},{"index":2,"sense_key":"present%2:40:02::","synset_id":"award.v.01"},{"index":3,"sense_key":"present%2:32:02::","synset_id":"deliver.v.01"},{"index":4,"sense_key":"present%2:32:06::","synset_id":"confront.v.03"},{"index":5,"sense_key":"present%2:40:01::","synset_id":"present.v.04"},{"index":6,"sense_key":"present%2:42:00::","synset_id":"present.v.05"},{"index":7,"sense_key":"present%2:36:08::","synset_id":"portray.v.04"},{"index":8,"sense_key":"present%2:32:04::","synset_id":"present.v.12"},{"index":9,"sense_key":"present%2:32:01::","synset_id":"introduce.v.01"},{"index":10,"sense_key":"present%2:36:00::","synset_id":"stage.v.01"},{"index":11,"sense_key":"present%2:32:00::","synset_id":"present.v.02"},{"index":12,"sense_key":"present%2:32:03::","synset_id":"salute.v.06"},{"index":13,"sense_key":"present%2:40:00::","synset_id":"give.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"present%2:39:00::","call_ids":["semeval2015.d002.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"present%2:39:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":959,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020635,"input_uncached_usd":0.004795000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.068967500003055}
{"item_id":"semeval2015.d002.s009.t007","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":2,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"policy%1:09:00::","call_ids":["semeval2015.d002.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.004765,"input_uncached_usd":0.002605,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.123868166934699}
{"item_id":"semeval2015.d002.s009.t008","gold_sense_keys":["practice%1:04:00::"],"candidates":[{"index":1,"sense_key":"practice%1:04:02::","synset_id":"exercise.n.03"},{"index":2,"sense_key":"practice%1:04:00::","synset_id":"practice.n.01"},{"index":3,"sense_key":"practice%1:04:04::","synset_id":"practice.n.03"},{"index":4,"sense_key":"practice%1:04:03::","synset_id":"practice.n.04"},{"index":5,"sense_key":"practice%1:09:00::","synset_id":"practice.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"practice%1:04:00::","call_ids":["semeval2015.d002.s009.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"practice%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":166,"reasoning_output_tokens":150},"cost":{"currency":"USD","total_usd":0.007985,"input_uncached_usd":0.0030050000000000003,"input_cached_usd":0.0,"output_usd":0.00498,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7277589170262218}
{"item_id":"semeval2015.d002.s009.t009","gold_sense_keys":["improve%2:30:01::"],"candidates":[{"index":1,"sense_key":"improve%2:30:00::","synset_id":"better.v.03"},{"index":2,"sense_key":"improve%2:30:01::","synset_id":"better.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"improve%2:30:01::","call_ids":["semeval2015.d002.s009.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"improve%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.00508,"input_uncached_usd":0.00238,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7685665000462905}
{"item_id":"semeval2015.d002.s009.t011","gold_sense_keys":["training%1:04:00::"],"candidates":[{"index":1,"sense_key":"training%1:04:00::","synset_id":"training.n.01"},{"index":2,"sense_key":"training%1:07:00::","synset_id":"education.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"training%1:04:00::","call_ids":["semeval2015.d002.s009.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"training%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":463,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.003965,"input_uncached_usd":0.0023150000000000002,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2934469160391018}
{"item_id":"semeval2015.d002.s009.t012","gold_sense_keys":["development%1:04:01::"],"candidates":[{"index":1,"sense_key":"development%1:10:00::","synset_id":"development.n.09"},{"index":2,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":3,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":4,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":5,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":6,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":7,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":8,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":9,"sense_key":"development%1:22:02::","synset_id":"development.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"development%1:04:01::","call_ids":["semeval2015.d002.s009.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"development%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":867,"cached_input_tokens":0,"output_tokens":216,"reasoning_output_tokens":200},"cost":{"currency":"USD","total_usd":0.010815000000000002,"input_uncached_usd":0.004335,"input_cached_usd":0.0,"output_usd":0.0064800000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.047990207909606}
{"item_id":"semeval2015.d002.s009.t014","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"},{"index":2,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"},{"index":3,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"},{"index":4,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worker%1:18:00::","call_ids":["semeval2015.d002.s009.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":73},"cost":{"currency":"USD","total_usd":0.00545,"input_uncached_usd":0.0027800000000000004,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.059863874921575}
{"item_id":"semeval2015.d002.s009.t017","gold_sense_keys":["community%1:14:00::"],"candidates":[{"index":1,"sense_key":"community%1:14:04::","synset_id":"community.n.03"},{"index":2,"sense_key":"community%1:14:02::","synset_id":"community.n.06"},{"index":3,"sense_key":"community%1:14:00::","synset_id":"community.n.01"},{"index":4,"sense_key":"community%1:15:00::","synset_id":"residential_district.n.01"},{"index":5,"sense_key":"community%1:21:00::","synset_id":"community.n.02"},{"index":6,"sense_key":"community%1:26:00::","synset_id":"community.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"community%1:14:00::","call_ids":["semeval2015.d002.s009.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"community%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":253,"reasoning_output_tokens":237},"cost":{"currency":"USD","total_usd":0.01064,"input_uncached_usd":0.00305,"input_cached_usd":0.0,"output_usd":0.00759,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.514855624991469}
{"item_id":"semeval2015.d002.s010.t000","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":2,"sense_key":"country%1:15:02::","synset_id":"area.n.01"},{"index":3,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":4,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":5,"sense_key":"country%1:15:01::","synset_id":"country.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"country%1:15:00::","call_ids":["semeval2015.d002.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"country%1:15:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01898,"input_uncached_usd":0.0031400000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.431729457923211}
{"item_id":"semeval2015.d002.s010.t005","gold_sense_keys":["case_study%1:10:00::"],"candidates":[{"index":1,"sense_key":"case_study%1:04:00::","synset_id":"case_study.n.02"},{"index":2,"sense_key":"case_study%1:10:00::","synset_id":"case_study.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"case_study%1:10:00::","call_ids":["semeval2015.d002.s010.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"case_study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":147},"cost":{"currency":"USD","total_usd":0.007185,"input_uncached_usd":0.0023250000000000002,"input_cached_usd":0.0,"output_usd":0.00486,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.030908290995285}
{"item_id":"semeval2015.d002.s011.t000","gold_sense_keys":["case_study%1:10:00::"],"candidates":[{"index":1,"sense_key":"case_study%1:10:00::","synset_id":"case_study.n.01"},{"index":2,"sense_key":"case_study%1:04:00::","synset_id":"case_study.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case_study%1:10:00::","call_ids":["semeval2015.d002.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case_study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":118},"cost":{"currency":"USD","total_usd":0.006215,"input_uncached_usd":0.002225,"input_cached_usd":0.0,"output_usd":0.0039900000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.1438757920404896}
{"item_id":"semeval2015.d002.s011.t001","gold_sense_keys":["analyse%2:31:00::","analyse%2:31:04::"],"candidates":[{"index":1,"sense_key":"analyse%2:31:01::","synset_id":"analyze.v.02"},{"index":2,"sense_key":"analyse%2:31:00::","synset_id":"analyze.v.01"},{"index":3,"sense_key":"analyse%2:31:02::","synset_id":"analyze.v.04"},{"index":4,"sense_key":"analyse%2:31:04::","synset_id":"analyze.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"analyse%2:31:00::","call_ids":["semeval2015.d002.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"analyse%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.005390000000000001,"input_uncached_usd":0.00284,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8304666250478476}
{"item_id":"semeval2015.d002.s011.t002","gold_sense_keys":["background%1:09:01::","background%1:26:00::"],"candidates":[{"index":1,"sense_key":"background%1:06:00::","synset_id":"backdrop.n.01"},{"index":2,"sense_key":"background%1:09:00::","synset_id":"background.n.02"},{"index":3,"sense_key":"background%1:11:00::","synset_id":"background.n.04"},{"index":4,"sense_key":"background%1:11:01::","synset_id":"background.n.05"},{"index":5,"sense_key":"background%1:07:00::","synset_id":"background.n.01"},{"index":6,"sense_key":"background%1:09:01::","synset_id":"background.n.03"},{"index":7,"sense_key":"background%1:26:00::","synset_id":"setting.n.02"},{"index":8,"sense_key":"background%1:06:01::","synset_id":"background.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"background%1:09:01::","call_ids":["semeval2015.d002.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"background%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":706,"cached_input_tokens":0,"output_tokens":225,"reasoning_output_tokens":209},"cost":{"currency":"USD","total_usd":0.010280000000000001,"input_uncached_usd":0.00353,"input_cached_usd":0.0,"output_usd":0.00675,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.497588917030953}
{"item_id":"semeval2015.d002.s011.t004","gold_sense_keys":["impact%1:19:00::"],"candidates":[{"index":1,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":2,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":3,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"},{"index":4,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"impact%1:19:00::","call_ids":["semeval2015.d002.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"impact%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":80},"cost":{"currency":"USD","total_usd":0.005475000000000001,"input_uncached_usd":0.002595,"input_cached_usd":0.0,"output_usd":0.00288,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1118165419902653}
{"item_id":"semeval2015.d002.s011.t005","gold_sense_keys":["employment%1:04:00::","employment%1:04:02::","employment%1:26:00::"],"candidates":[{"index":1,"sense_key":"employment%1:04:00::","synset_id":"employment.n.02"},{"index":2,"sense_key":"employment%1:04:01::","synset_id":"use.n.01"},{"index":3,"sense_key":"employment%1:04:02::","synset_id":"employment.n.03"},{"index":4,"sense_key":"employment%1:26:00::","synset_id":"employment.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"employment%1:04:00::","call_ids":["semeval2015.d002.s011.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"employment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018505,"input_uncached_usd":0.0026650000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.924989415914752}
{"item_id":"semeval2015.d002.s011.t007","gold_sense_keys":["work%1:04:01::"],"candidates":[{"index":1,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":2,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":3,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"},{"index":4,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":5,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":6,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":7,"sense_key":"work%1:09:00::","synset_id":"study.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"work%1:04:00::","call_ids":["semeval2015.d002.s011.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"work%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":720,"cached_input_tokens":0,"output_tokens":1544,"reasoning_output_tokens":1515},"cost":{"currency":"USD","total_usd":0.049920000000000006,"input_uncached_usd":0.0036000000000000003,"input_cached_usd":0.0,"output_usd":0.04632,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":35.26736083405558}
{"item_id":"semeval2015.d002.s011.t009","gold_sense_keys":["assess%2:31:00::"],"candidates":[{"index":1,"sense_key":"assess%2:40:00::","synset_id":"tax.v.02"},{"index":2,"sense_key":"assess%2:31:01::","synset_id":"assess.v.04"},{"index":3,"sense_key":"assess%2:31:00::","synset_id":"measure.v.04"},{"index":4,"sense_key":"assess%2:40:01::","synset_id":"assess.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"assess%2:31:00::","call_ids":["semeval2015.d002.s011.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"assess%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":539,"cached_input_tokens":0,"output_tokens":38,"reasoning_output_tokens":23},"cost":{"currency":"USD","total_usd":0.0038350000000000003,"input_uncached_usd":0.0026950000000000003,"input_cached_usd":0.0,"output_usd":0.00114,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3437400839757174}
{"item_id":"semeval2015.d002.s011.t010","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":2,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"},{"index":3,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"benefit%1:07:00::","call_ids":["semeval2015.d002.s011.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.004605,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8489410410402343}
{"item_id":"semeval2015.d002.s011.t011","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"},{"index":2,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":3,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"problem%1:09:00::","call_ids":["semeval2015.d002.s011.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"problem%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":109},"cost":{"currency":"USD","total_usd":0.006125,"input_uncached_usd":0.002405,"input_cached_usd":0.0,"output_usd":0.00372,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9569207499735057}
{"item_id":"semeval2015.d002.s012.t000","gold_sense_keys":["visit%1:04:01::"],"candidates":[{"index":1,"sense_key":"visit%1:04:00::","synset_id":"visit.n.04"},{"index":2,"sense_key":"visit%1:04:03::","synset_id":"sojourn.n.01"},{"index":3,"sense_key":"visit%1:04:02::","synset_id":"visit.n.01"},{"index":4,"sense_key":"visit%1:04:01::","synset_id":"visit.n.03"},{"index":5,"sense_key":"visit%1:14:00::","synset_id":"visit.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"visit%1:04:01::","call_ids":["semeval2015.d002.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"visit%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018515,"input_uncached_usd":0.0026750000000000003,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.673290124977939}
{"item_id":"semeval2015.d002.s012.t001","gold_sense_keys":["interview%1:10:01::"],"candidates":[{"index":1,"sense_key":"interview%1:10:00::","synset_id":"consultation.n.01"},{"index":2,"sense_key":"interview%1:10:01::","synset_id":"interview.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"interview%1:10:01::","call_ids":["semeval2015.d002.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"interview%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":71},"cost":{"currency":"USD","total_usd":0.004855,"input_uncached_usd":0.002245,"input_cached_usd":0.0,"output_usd":0.00261,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9388423750642687}
{"item_id":"semeval2015.d002.s013.t000","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":2,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":3,"sense_key":"national%3:01:01::","synset_id":"national.a.07"},{"index":4,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":5,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":6,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":7,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"national%3:00:01::","call_ids":["semeval2015.d002.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":836,"reasoning_output_tokens":814},"cost":{"currency":"USD","total_usd":0.028235000000000003,"input_uncached_usd":0.0031550000000000003,"input_cached_usd":0.0,"output_usd":0.02508,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.767919625039212}
{"item_id":"semeval2015.d002.s013.t001","gold_sense_keys":["study%1:10:00::"],"candidates":[{"index":1,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":2,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":3,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":4,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":5,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":6,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":7,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":8,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":9,"sense_key":"study%1:10:01::","synset_id":"study.n.10"},{"index":10,"sense_key":"study%1:09:04::","synset_id":"study.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d002.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":779,"cached_input_tokens":0,"output_tokens":468,"reasoning_output_tokens":452},"cost":{"currency":"USD","total_usd":0.017935,"input_uncached_usd":0.0038950000000000005,"input_cached_usd":0.0,"output_usd":0.01404,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.38004762493074}
{"item_id":"semeval2015.d002.s013.t002","gold_sense_keys":["form%2:42:00::"],"candidates":[{"index":1,"sense_key":"form%2:42:02::","synset_id":"form.v.03"},{"index":2,"sense_key":"form%2:30:01::","synset_id":"shape.v.03"},{"index":3,"sense_key":"form%2:41:00::","synset_id":"form.v.01"},{"index":4,"sense_key":"form%2:41:02::","synset_id":"imprint.v.01"},{"index":5,"sense_key":"form%2:42:00::","synset_id":"form.v.02"},{"index":6,"sense_key":"form%2:36:00::","synset_id":"shape.v.02"},{"index":7,"sense_key":"form%2:30:05::","synset_id":"form.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"form%2:42:00::","call_ids":["semeval2015.d002.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"form%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":68},"cost":{"currency":"USD","total_usd":0.005755000000000001,"input_uncached_usd":0.0032350000000000005,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2329997919732705}
{"item_id":"semeval2015.d002.s013.t003","gold_sense_keys":["basis%1:09:00::"],"candidates":[{"index":1,"sense_key":"basis%1:24:00::","synset_id":"footing.n.02"},{"index":2,"sense_key":"basis%1:24:01::","synset_id":"basis.n.03"},{"index":3,"sense_key":"basis%1:09:00::","synset_id":"basis.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"basis%1:09:00::","call_ids":["semeval2015.d002.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"basis%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":507,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":130},"cost":{"currency":"USD","total_usd":0.006915000000000001,"input_uncached_usd":0.0025350000000000004,"input_cached_usd":0.0,"output_usd":0.00438,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.367530334042385}
{"item_id":"semeval2015.d002.s013.t005","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":3,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":4,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":5,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:11:00::","synset_id":"report.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2015.d002.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":689,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.005695,"input_uncached_usd":0.003445,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.037013166001998}
{"item_id":"semeval2015.d002.s013.t007","gold_sense_keys":["overall%5:00:00:general:00"],"candidates":[{"index":1,"sense_key":"overall%5:00:00:gross:00","synset_id":"overall.s.02"},{"index":2,"sense_key":"overall%5:00:00:general:00","synset_id":"overall.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"overall%5:00:00:general:00","call_ids":["semeval2015.d002.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"overall%5:00:00:general:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":425,"cached_input_tokens":0,"output_tokens":335,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.012175000000000002,"input_uncached_usd":0.002125,"input_cached_usd":0.0,"output_usd":0.01005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.954098375048488}
{"item_id":"semeval2015.d002.s013.t008","gold_sense_keys":["practice%1:04:00::"],"candidates":[{"index":1,"sense_key":"practice%1:09:00::","synset_id":"practice.n.05"},{"index":2,"sense_key":"practice%1:04:00::","synset_id":"practice.n.01"},{"index":3,"sense_key":"practice%1:04:02::","synset_id":"exercise.n.03"},{"index":4,"sense_key":"practice%1:04:04::","synset_id":"practice.n.03"},{"index":5,"sense_key":"practice%1:04:03::","synset_id":"practice.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"practice%1:04:00::","call_ids":["semeval2015.d002.s013.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"practice%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":304,"reasoning_output_tokens":288},"cost":{"currency":"USD","total_usd":0.01199,"input_uncached_usd":0.00287,"input_cached_usd":0.0,"output_usd":0.00912,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.236508124973625}
{"item_id":"semeval2015.d002.s014.t000","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":2,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"},{"index":3,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":4,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":5,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":6,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":7,"sense_key":"report%1:11:00::","synset_id":"report.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2015.d002.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":641,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.004855,"input_uncached_usd":0.0032050000000000004,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.512082041008398}
{"item_id":"semeval2015.d002.s014.t003","gold_sense_keys":["publish%2:32:00::","publish%2:36:01::"],"candidates":[{"index":1,"sense_key":"publish%2:32:00::","synset_id":"publish.v.02"},{"index":2,"sense_key":"publish%2:36:00::","synset_id":"publish.v.03"},{"index":3,"sense_key":"publish%2:36:01::","synset_id":"print.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"publish%2:32:00::","call_ids":["semeval2015.d002.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"publish%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":256,"reasoning_output_tokens":240},"cost":{"currency":"USD","total_usd":0.009980000000000001,"input_uncached_usd":0.0023000000000000004,"input_cached_usd":0.0,"output_usd":0.00768,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.166956541943364}
{"item_id":"semeval2015.d002.s014.t005","gold_sense_keys":["conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"},{"index":2,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":3,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"conference%1:14:00::","call_ids":["semeval2015.d002.s014.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":65},"cost":{"currency":"USD","total_usd":0.004490000000000001,"input_uncached_usd":0.0020900000000000003,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8789656249573454}
{"item_id":"semeval2015.d002.s014.t006","gold_sense_keys":["stage%2:36:01::"],"candidates":[{"index":1,"sense_key":"stage%2:36:01::","synset_id":"stage.v.02"},{"index":2,"sense_key":"stage%2:36:00::","synset_id":"stage.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stage%2:36:01::","call_ids":["semeval2015.d002.s014.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stage%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":49},"cost":{"currency":"USD","total_usd":0.003935,"input_uncached_usd":0.0020150000000000003,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5439152910839766}
{"item_id":"semeval2015.d002.s015.t000","gold_sense_keys":["foundation%1:14:00::"],"candidates":[{"index":1,"sense_key":"foundation%1:14:00::","synset_id":"foundation.n.02"},{"index":2,"sense_key":"foundation%1:09:01::","synset_id":"foundation.n.04"},{"index":3,"sense_key":"foundation%1:06:00::","synset_id":"foundation.n.03"},{"index":4,"sense_key":"foundation%1:06:01::","synset_id":"foundation_garment.n.01"},{"index":5,"sense_key":"foundation%1:09:00::","synset_id":"basis.n.02"},{"index":6,"sense_key":"foundation%1:04:00::","synset_id":"initiation.n.02"},{"index":7,"sense_key":"foundation%1:24:00::","synset_id":"foundation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"foundation%1:14:00::","call_ids":["semeval2015.d002.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"foundation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":96},"cost":{"currency":"USD","total_usd":0.006435,"input_uncached_usd":0.003105,"input_cached_usd":0.0,"output_usd":0.00333,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.0161171249346808}
{"item_id":"semeval2015.d002.s015.t001","gold_sense_keys":["aim%2:31:01::"],"candidates":[{"index":1,"sense_key":"aim%2:31:01::","synset_id":"aim.v.02"},{"index":2,"sense_key":"aim%2:33:10::","synset_id":"target.v.01"},{"index":3,"sense_key":"aim%2:33:00::","synset_id":"aim.v.01"},{"index":4,"sense_key":"aim%2:32:00::","synset_id":"aim.v.06"},{"index":5,"sense_key":"aim%2:31:00::","synset_id":"draw_a_bead_on.v.02"},{"index":6,"sense_key":"aim%2:32:09::","synset_id":"drive.v.11"},{"index":7,"sense_key":"aim%2:31:02::","synset_id":"calculate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"aim%2:31:01::","call_ids":["semeval2015.d002.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"aim%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.00476,"input_uncached_usd":0.00302,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7794867089251056}
{"item_id":"semeval2015.d002.s015.t002","gold_sense_keys":["document%2:32:00::"],"candidates":[{"index":1,"sense_key":"document%2:32:00::","synset_id":"document.v.01"},{"index":2,"sense_key":"document%2:31:00::","synset_id":"document.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"document%2:32:00::","call_ids":["semeval2015.d002.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"document%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":356,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":86},"cost":{"currency":"USD","total_usd":0.00481,"input_uncached_usd":0.0017800000000000001,"input_cached_usd":0.0,"output_usd":0.00303,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.3170210419921204}
{"item_id":"semeval2015.d002.s015.t003","gold_sense_keys":["characteristic%1:07:00::","characteristic%1:09:00::"],"candidates":[{"index":1,"sense_key":"characteristic%1:07:01::","synset_id":"characteristic.n.04"},{"index":2,"sense_key":"characteristic%1:10:00::","synset_id":"characteristic.n.03"},{"index":3,"sense_key":"characteristic%1:09:00::","synset_id":"feature.n.01"},{"index":4,"sense_key":"characteristic%1:07:00::","synset_id":"characteristic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"characteristic%1:09:00::","call_ids":["semeval2015.d002.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"characteristic%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":298,"reasoning_output_tokens":282},"cost":{"currency":"USD","total_usd":0.01117,"input_uncached_usd":0.00223,"input_cached_usd":0.0,"output_usd":0.00894,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.995907707954757}
{"item_id":"semeval2015.d002.s015.t004","gold_sense_keys":["age%1:07:00::"],"candidates":[{"index":1,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"},{"index":2,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"},{"index":3,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":4,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":5,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"age%1:07:00::","call_ids":["semeval2015.d002.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"age%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.033964999999999995,"input_uncached_usd":0.002585,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":20.72232741699554}
{"item_id":"semeval2015.d002.s015.t005","gold_sense_keys":["barrier%1:09:00::"],"candidates":[{"index":1,"sense_key":"barrier%1:06:00::","synset_id":"barrier.n.01"},{"index":2,"sense_key":"barrier%1:09:00::","synset_id":"barrier.n.02"},{"index":3,"sense_key":"barrier%1:17:00::","synset_id":"barrier.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"barrier%1:09:00::","call_ids":["semeval2015.d002.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"barrier%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":398,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00379,"input_uncached_usd":0.00199,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.791468666982837}
{"item_id":"semeval2015.d002.s015.t006","gold_sense_keys":["job%1:04:00::"],"candidates":[{"index":1,"sense_key":"job%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"job%1:04:01::","synset_id":"job.n.02"},{"index":3,"sense_key":"job%1:04:02::","synset_id":"caper.n.03"},{"index":4,"sense_key":"job%1:10:00::","synset_id":"job.n.12"},{"index":5,"sense_key":"job%1:04:03::","synset_id":"job.n.07"},{"index":6,"sense_key":"job%1:04:05::","synset_id":"job.n.06"},{"index":7,"sense_key":"job%1:18:00::","synset_id":"job.n.09"},{"index":8,"sense_key":"job%1:04:00::","synset_id":"occupation.n.01"},{"index":9,"sense_key":"job%1:04:04::","synset_id":"job.n.05"},{"index":10,"sense_key":"job%1:18:01::","synset_id":"job.n.10"},{"index":11,"sense_key":"job%1:10:01::","synset_id":"job.n.11"},{"index":12,"sense_key":"job%1:06:00::","synset_id":"job.n.03"},{"index":13,"sense_key":"job%1:06:01::","synset_id":"job.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"job%1:04:00::","call_ids":["semeval2015.d002.s015.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"job%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":845,"cached_input_tokens":0,"output_tokens":236,"reasoning_output_tokens":220},"cost":{"currency":"USD","total_usd":0.011305,"input_uncached_usd":0.0042250000000000005,"input_cached_usd":0.0,"output_usd":0.00708,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.158007874968462}
{"item_id":"semeval2015.d002.s015.t008","gold_sense_keys":["training%1:04:00::"],"candidates":[{"index":1,"sense_key":"training%1:04:00::","synset_id":"training.n.01"},{"index":2,"sense_key":"training%1:07:00::","synset_id":"education.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"training%1:04:00::","call_ids":["semeval2015.d002.s015.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"training%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":369,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.0034349999999999997,"input_uncached_usd":0.001845,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2647192919394}
{"item_id":"semeval2015.d002.s015.t009","gold_sense_keys":["analyse%2:31:00::","analyse%2:31:04::"],"candidates":[{"index":1,"sense_key":"analyse%2:31:02::","synset_id":"analyze.v.04"},{"index":2,"sense_key":"analyse%2:31:00::","synset_id":"analyze.v.01"},{"index":3,"sense_key":"analyse%2:31:04::","synset_id":"analyze.v.03"},{"index":4,"sense_key":"analyse%2:31:01::","synset_id":"analyze.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"analyse%2:31:00::","call_ids":["semeval2015.d002.s015.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"analyse%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":87},"cost":{"currency":"USD","total_usd":0.005575,"input_uncached_usd":0.0025150000000000003,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.426352374954149}
{"item_id":"semeval2015.d002.s015.t010","gold_sense_keys":["cause%1:03:00::","cause%1:11:00::"],"candidates":[{"index":1,"sense_key":"cause%1:04:00::","synset_id":"lawsuit.n.01"},{"index":2,"sense_key":"cause%1:04:01::","synset_id":"campaign.n.02"},{"index":3,"sense_key":"cause%1:11:00::","synset_id":"cause.n.01"},{"index":4,"sense_key":"cause%1:10:00::","synset_id":"cause.n.02"},{"index":5,"sense_key":"cause%1:03:00::","synset_id":"causal_agent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cause%1:11:00::","call_ids":["semeval2015.d002.s015.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cause%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":419,"reasoning_output_tokens":403},"cost":{"currency":"USD","total_usd":0.015255,"input_uncached_usd":0.0026850000000000003,"input_cached_usd":0.0,"output_usd":0.01257,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.730926541960798}
{"item_id":"semeval2015.d002.s016.t000","gold_sense_keys":["perspective%1:09:00::"],"candidates":[{"index":1,"sense_key":"perspective%1:07:00::","synset_id":"perspective.n.02"},{"index":2,"sense_key":"perspective%1:09:00::","synset_id":"position.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"perspective%1:09:00::","call_ids":["semeval2015.d002.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"perspective%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":388,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.00371,"input_uncached_usd":0.00194,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6962192920036614}
{"item_id":"semeval2015.d002.s016.t001","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":2,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"},{"index":3,"sense_key":"social%3:00:00::","synset_id":"social.a.02"},{"index":4,"sense_key":"social%3:01:00::","synset_id":"social.a.01"},{"index":5,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"},{"index":6,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":201,"reasoning_output_tokens":186},"cost":{"currency":"USD","total_usd":0.00881,"input_uncached_usd":0.0027800000000000004,"input_cached_usd":0.0,"output_usd":0.00603,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.9609680409776047}
{"item_id":"semeval2015.d002.s016.t004","gold_sense_keys":["particular%5:00:00:uncommon:00"],"candidates":[{"index":1,"sense_key":"particular%5:00:00:fastidious:00","synset_id":"finical.s.01"},{"index":2,"sense_key":"particular%5:00:00:careful:00","synset_id":"particular.s.06"},{"index":3,"sense_key":"particular%5:00:02:specific:00","synset_id":"particular.s.02"},{"index":4,"sense_key":"particular%5:00:00:primary:00","synset_id":"particular.s.04"},{"index":5,"sense_key":"particular%5:00:00:specific:00","synset_id":"particular.s.01"},{"index":6,"sense_key":"particular%5:00:00:uncommon:00","synset_id":"especial.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"particular%5:00:00:uncommon:00","call_ids":["semeval2015.d002.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"particular%5:00:00:uncommon:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":376,"reasoning_output_tokens":360},"cost":{"currency":"USD","total_usd":0.01431,"input_uncached_usd":0.00303,"input_cached_usd":0.0,"output_usd":0.01128,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.436519749928266}
{"item_id":"semeval2015.d002.s016.t005","gold_sense_keys":["attention%1:09:00::"],"candidates":[{"index":1,"sense_key":"attention%1:09:00::","synset_id":"attention.n.01"},{"index":2,"sense_key":"attention%1:04:02::","synset_id":"attention.n.04"},{"index":3,"sense_key":"attention%1:09:02::","synset_id":"attention.n.03"},{"index":4,"sense_key":"attention%1:09:01::","synset_id":"attention.n.05"},{"index":5,"sense_key":"attention%1:07:00::","synset_id":"attention.n.06"},{"index":6,"sense_key":"attention%1:04:01::","synset_id":"care.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attention%1:09:00::","call_ids":["semeval2015.d002.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attention%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":182,"reasoning_output_tokens":167},"cost":{"currency":"USD","total_usd":0.00822,"input_uncached_usd":0.0027600000000000003,"input_cached_usd":0.0,"output_usd":0.0054600000000000004,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.605461374972947}
{"item_id":"semeval2015.d002.s016.t006","gold_sense_keys":["pay%2:31:00::","pay%2:32:00::"],"candidates":[{"index":1,"sense_key":"pay%2:42:00::","synset_id":"pay.v.07"},{"index":2,"sense_key":"pay%2:31:00::","synset_id":"give.v.10"},{"index":3,"sense_key":"pay%2:40:14::","synset_id":"pay.v.11"},{"index":4,"sense_key":"pay%2:31:03::","synset_id":"pay.v.09"},{"index":5,"sense_key":"pay%2:40:04::","synset_id":"pay_up.v.01"},{"index":6,"sense_key":"pay%2:40:01::","synset_id":"yield.v.10"},{"index":7,"sense_key":"pay%2:40:02::","synset_id":"pay.v.05"},{"index":8,"sense_key":"pay%2:40:13::","synset_id":"pay.v.10"},{"index":9,"sense_key":"pay%2:41:00::","synset_id":"pay.v.08"},{"index":10,"sense_key":"pay%2:32:00::","synset_id":"give.v.05"},{"index":11,"sense_key":"pay%2:40:00::","synset_id":"pay.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"pay%2:32:00::","call_ids":["semeval2015.d002.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"pay%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":726,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01947,"input_uncached_usd":0.0036300000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.949889875017107}
{"item_id":"semeval2015.d002.s016.t007","gold_sense_keys":["proposal%1:10:00::"],"candidates":[{"index":1,"sense_key":"proposal%1:10:02::","synset_id":"marriage_proposal.n.01"},{"index":2,"sense_key":"proposal%1:10:00::","synset_id":"proposal.n.01"},{"index":3,"sense_key":"proposal%1:10:01::","synset_id":"proposal.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"proposal%1:10:00::","call_ids":["semeval2015.d002.s016.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"proposal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.0039900000000000005,"input_uncached_usd":0.00201,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6194257080787793}
{"item_id":"semeval2015.d002.s016.t008","gold_sense_keys":["remove%2:30:00::","remove%2:40:00::"],"candidates":[{"index":1,"sense_key":"remove%2:30:00::","synset_id":"remove.v.01"},{"index":2,"sense_key":"remove%2:41:00::","synset_id":"remove.v.02"},{"index":3,"sense_key":"remove%2:40:00::","synset_id":"get_rid_of.v.01"},{"index":4,"sense_key":"remove%2:30:02::","synset_id":"remove.v.08"},{"index":5,"sense_key":"remove%2:30:01::","synset_id":"absent.v.01"},{"index":6,"sense_key":"remove%2:41:02::","synset_id":"take_out.v.01"},{"index":7,"sense_key":"remove%2:38:00::","synset_id":"remove.v.05"},{"index":8,"sense_key":"remove%2:41:01::","synset_id":"murder.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"remove%2:30:02::","call_ids":["semeval2015.d002.s016.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"remove%2:30:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":657,"cached_input_tokens":0,"output_tokens":191,"reasoning_output_tokens":175},"cost":{"currency":"USD","total_usd":0.009015,"input_uncached_usd":0.003285,"input_cached_usd":0.0,"output_usd":0.00573,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.609110291930847}
{"item_id":"semeval2015.d002.s016.t009","gold_sense_keys":["age%1:07:00::"],"candidates":[{"index":1,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"},{"index":2,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"},{"index":3,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":4,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":5,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"age%1:28:00::","call_ids":["semeval2015.d002.s016.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"age%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":262,"reasoning_output_tokens":246},"cost":{"currency":"USD","total_usd":0.010490000000000001,"input_uncached_usd":0.0026300000000000004,"input_cached_usd":0.0,"output_usd":0.00786,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.07919579197187}
{"item_id":"semeval2015.d002.s016.t010","gold_sense_keys":["barrier%1:09:00::"],"candidates":[{"index":1,"sense_key":"barrier%1:06:00::","synset_id":"barrier.n.01"},{"index":2,"sense_key":"barrier%1:09:00::","synset_id":"barrier.n.02"},{"index":3,"sense_key":"barrier%1:17:00::","synset_id":"barrier.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"barrier%1:09:00::","call_ids":["semeval2015.d002.s016.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"barrier%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":406,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.00395,"input_uncached_usd":0.00203,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.741866249940358}
{"item_id":"semeval2015.d002.s017.t002","gold_sense_keys":["assess%2:31:00::"],"candidates":[{"index":1,"sense_key":"assess%2:31:01::","synset_id":"assess.v.04"},{"index":2,"sense_key":"assess%2:40:00::","synset_id":"tax.v.02"},{"index":3,"sense_key":"assess%2:31:00::","synset_id":"measure.v.04"},{"index":4,"sense_key":"assess%2:40:01::","synset_id":"assess.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"assess%2:31:00::","call_ids":["semeval2015.d002.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"assess%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":40,"reasoning_output_tokens":24},"cost":{"currency":"USD","total_usd":0.003615,"input_uncached_usd":0.002415,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6454807500122115}
{"item_id":"semeval2015.d002.s017.t005","gold_sense_keys":["private%3:00:00::"],"candidates":[{"index":1,"sense_key":"private%5:00:00:personal:00","synset_id":"individual.s.03"},{"index":2,"sense_key":"private%5:00:02:personal:00","synset_id":"private.s.01"},{"index":3,"sense_key":"private%3:00:00::","synset_id":"private.a.01"},{"index":4,"sense_key":"private%5:00:00:inward:00","synset_id":"secret.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"private%3:00:00::","call_ids":["semeval2015.d002.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"private%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":132},"cost":{"currency":"USD","total_usd":0.006710000000000001,"input_uncached_usd":0.0022700000000000003,"input_cached_usd":0.0,"output_usd":0.00444,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.015364290913567}
{"item_id":"semeval2015.d002.s017.t006","gold_sense_keys":["sector%1:14:00::"],"candidates":[{"index":1,"sense_key":"sector%1:06:00::","synset_id":"sector.n.06"},{"index":2,"sense_key":"sector%1:14:00::","synset_id":"sector.n.02"},{"index":3,"sense_key":"sector%1:23:00::","synset_id":"sector.n.04"},{"index":4,"sense_key":"sector%1:25:00::","synset_id":"sector.n.01"},{"index":5,"sense_key":"sector%1:15:00::","synset_id":"sector.n.05"},{"index":6,"sense_key":"sector%1:09:00::","synset_id":"sector.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sector%1:14:00::","call_ids":["semeval2015.d002.s017.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sector%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.004365,"input_uncached_usd":0.0025050000000000003,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.482297500013374}
{"item_id":"semeval2015.d002.s017.t007","gold_sense_keys":["age%1:07:00::"],"candidates":[{"index":1,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"},{"index":2,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":3,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"},{"index":4,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":5,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"age%1:07:00::","call_ids":["semeval2015.d002.s017.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"age%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01846,"input_uncached_usd":0.0026200000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.083990415907465}
{"item_id":"semeval2015.d002.s017.t008","gold_sense_keys":["barrier%1:09:00::"],"candidates":[{"index":1,"sense_key":"barrier%1:17:00::","synset_id":"barrier.n.03"},{"index":2,"sense_key":"barrier%1:09:00::","synset_id":"barrier.n.02"},{"index":3,"sense_key":"barrier%1:06:00::","synset_id":"barrier.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"barrier%1:09:00::","call_ids":["semeval2015.d002.s017.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"barrier%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.00427,"input_uncached_usd":0.00202,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9556170420255512}
{"item_id":"semeval2015.d002.s017.t009","gold_sense_keys":["especially%4:02:00::"],"candidates":[{"index":1,"sense_key":"especially%4:02:01::","synset_id":"specially.r.01"},{"index":2,"sense_key":"especially%4:02:00::","synset_id":"particularly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"especially%4:02:00::","call_ids":["semeval2015.d002.s017.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"especially%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":386,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.00394,"input_uncached_usd":0.00193,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7263466250151396}
{"item_id":"semeval2015.d002.s017.t010","gold_sense_keys":["older%5:00:00:experienced:00","older%5:00:00:old:02"],"candidates":[{"index":1,"sense_key":"erstwhile%5:00:00:past:00","synset_id":"erstwhile.s.01"},{"index":2,"sense_key":"older%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":3,"sense_key":"previous%5:00:00:preceding:00","synset_id":"previous.s.01"},{"index":4,"sense_key":"older%5:00:00:senior:00","synset_id":"elder.s.01"},{"index":5,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"},{"index":6,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"},{"index":7,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":8,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":9,"sense_key":"honest-to-god%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"},{"index":10,"sense_key":"older%5:00:00:old:02","synset_id":"aged.s.01"},{"index":11,"sense_key":"older%5:00:00:experienced:00","synset_id":"old.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"older%5:00:00:old:02","call_ids":["semeval2015.d002.s017.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"older%5:00:00:old:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":860,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.020139999999999998,"input_uncached_usd":0.0043,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.29833504196722}
{"item_id":"semeval2015.d002.s017.t011","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"},{"index":2,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"},{"index":3,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"},{"index":4,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"worker%1:18:00::","call_ids":["semeval2015.d002.s017.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":468,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":121},"cost":{"currency":"USD","total_usd":0.00645,"input_uncached_usd":0.00234,"input_cached_usd":0.0,"output_usd":0.00411,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.60426237503998}
{"item_id":"semeval2015.d002.s018.t000","gold_sense_keys":["seek%2:35:00::"],"candidates":[{"index":1,"sense_key":"seek%2:35:00::","synset_id":"search.v.01"},{"index":2,"sense_key":"seek%2:41:00::","synset_id":"try.v.01"},{"index":3,"sense_key":"seek%2:40:00::","synset_id":"seek.v.01"},{"index":4,"sense_key":"seek%2:38:00::","synset_id":"seek.v.04"},{"index":5,"sense_key":"seek%2:32:00::","synset_id":"seek.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seek%2:35:00::","call_ids":["semeval2015.d002.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seek%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01839,"input_uncached_usd":0.00255,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.531756791984662}
{"item_id":"semeval2015.d002.s018.t002","gold_sense_keys":["early%3:00:00::"],"candidates":[{"index":1,"sense_key":"early%3:00:00::","synset_id":"early.a.01"},{"index":2,"sense_key":"early%5:00:00:past:00","synset_id":"early.s.01"},{"index":3,"sense_key":"early%3:00:01::","synset_id":"early.a.05"},{"index":4,"sense_key":"early%3:00:02::","synset_id":"early.a.02"},{"index":5,"sense_key":"early%5:00:00:young:00","synset_id":"early.s.02"},{"index":6,"sense_key":"early%5:00:01:future:00","synset_id":"early.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"early%3:00:00::","call_ids":["semeval2015.d002.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"early%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.005180000000000001,"input_uncached_usd":0.00287,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8434264580719173}
{"item_id":"semeval2015.d002.s018.t003","gold_sense_keys":["retirement%1:04:00::","retirement%1:26:00::"],"candidates":[{"index":1,"sense_key":"retirement%1:26:00::","synset_id":"retirement.n.01"},{"index":2,"sense_key":"retirement%1:04:01::","synset_id":"retirement.n.03"},{"index":3,"sense_key":"retirement%1:04:00::","synset_id":"retirement.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"retirement%1:04:00::","call_ids":["semeval2015.d002.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"retirement%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.004375,"input_uncached_usd":0.0020350000000000004,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8544977080309764}
{"item_id":"semeval2015.d002.s018.t004","gold_sense_keys":["redundancy%1:07:00::"],"candidates":[{"index":1,"sense_key":"redundancy%1:10:00::","synset_id":"redundancy.n.01"},{"index":2,"sense_key":"redundancy%1:07:00::","synset_id":"redundancy.n.02"},{"index":3,"sense_key":"redundancy%1:09:00::","synset_id":"redundancy.n.03"},{"index":4,"sense_key":"redundancy%1:04:00::","synset_id":"redundancy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"redundancy%1:07:00::","call_ids":["semeval2015.d002.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"redundancy%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.004790000000000001,"input_uncached_usd":0.0022400000000000002,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0006997090531513}
{"item_id":"semeval2015.d002.s018.t007","gold_sense_keys":["concerned%5:00:00:involved:00"],"candidates":[{"index":1,"sense_key":"concerned%3:00:00::","synset_id":"concerned.a.01"},{"index":2,"sense_key":"concerned%5:00:00:involved:00","synset_id":"concerned.s.01"},{"index":3,"sense_key":"concerned%5:00:01:involved:00","synset_id":"implicated.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"concerned%5:00:00:involved:00","call_ids":["semeval2015.d002.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"concerned%5:00:00:involved:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":466,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":132},"cost":{"currency":"USD","total_usd":0.00677,"input_uncached_usd":0.00233,"input_cached_usd":0.0,"output_usd":0.00444,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8349237500224262}
{"item_id":"semeval2015.d002.s018.t008","gold_sense_keys":["retention%1:04:00::"],"candidates":[{"index":1,"sense_key":"retention%1:09:00::","synset_id":"memory.n.03"},{"index":2,"sense_key":"retention%1:07:00::","synset_id":"retentiveness.n.03"},{"index":3,"sense_key":"retention%1:04:00::","synset_id":"retention.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"retention%1:04:00::","call_ids":["semeval2015.d002.s018.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"retention%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.004295,"input_uncached_usd":0.0021650000000000003,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9240752089535818}
{"item_id":"semeval2015.d002.s018.t011","gold_sense_keys":["older%5:00:00:experienced:00","older%5:00:00:old:02"],"candidates":[{"index":1,"sense_key":"erstwhile%5:00:00:past:00","synset_id":"erstwhile.s.01"},{"index":2,"sense_key":"older%5:00:00:senior:00","synset_id":"elder.s.01"},{"index":3,"sense_key":"older%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":4,"sense_key":"previous%5:00:00:preceding:00","synset_id":"previous.s.01"},{"index":5,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"},{"index":6,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":7,"sense_key":"older%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":8,"sense_key":"honest-to-god%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"},{"index":9,"sense_key":"older%5:00:00:old:02","synset_id":"aged.s.01"},{"index":10,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":11,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"older%5:00:00:old:02","call_ids":["semeval2015.d002.s018.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"older%5:00:00:old:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":867,"cached_input_tokens":0,"output_tokens":210,"reasoning_output_tokens":194},"cost":{"currency":"USD","total_usd":0.010635000000000002,"input_uncached_usd":0.004335,"input_cached_usd":0.0,"output_usd":0.0063,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.049485916970298}
{"item_id":"semeval2015.d002.s018.t012","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"},{"index":2,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"},{"index":3,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"},{"index":4,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worker%1:18:00::","call_ids":["semeval2015.d002.s018.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":170,"reasoning_output_tokens":154},"cost":{"currency":"USD","total_usd":0.007475000000000001,"input_uncached_usd":0.0023750000000000004,"input_cached_usd":0.0,"output_usd":0.0051,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.055394290946424}
{"item_id":"semeval2015.d002.s019.t000","gold_sense_keys":["gender%1:07:00::"],"candidates":[{"index":1,"sense_key":"gender%1:07:00::","synset_id":"sex.n.04"},{"index":2,"sense_key":"gender%1:10:00::","synset_id":"gender.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"gender%1:07:00::","call_ids":["semeval2015.d002.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"gender%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.0038900000000000002,"input_uncached_usd":0.0021200000000000004,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4662785000400618}
{"item_id":"semeval2015.d002.s019.t001","gold_sense_keys":["ethnic%5:00:00:social:00"],"candidates":[{"index":1,"sense_key":"ethnic%5:00:00:social:00","synset_id":"cultural.s.01"},{"index":2,"sense_key":"ethnic%5:00:00:irreligious:00","synset_id":"heathen.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ethnic%5:00:00:social:00","call_ids":["semeval2015.d002.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ethnic%5:00:00:social:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.0037500000000000003,"input_uncached_usd":0.0021300000000000004,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2484387919539586}
{"item_id":"semeval2015.d002.s019.t002","gold_sense_keys":["background%1:07:00::"],"candidates":[{"index":1,"sense_key":"background%1:07:00::","synset_id":"background.n.01"},{"index":2,"sense_key":"background%1:26:00::","synset_id":"setting.n.02"},{"index":3,"sense_key":"background%1:11:01::","synset_id":"background.n.05"},{"index":4,"sense_key":"background%1:09:01::","synset_id":"background.n.03"},{"index":5,"sense_key":"background%1:06:01::","synset_id":"background.n.07"},{"index":6,"sense_key":"background%1:06:00::","synset_id":"backdrop.n.01"},{"index":7,"sense_key":"background%1:11:00::","synset_id":"background.n.04"},{"index":8,"sense_key":"background%1:09:00::","synset_id":"background.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"background%1:07:00::","call_ids":["semeval2015.d002.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"background%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":75},"cost":{"currency":"USD","total_usd":0.006005,"input_uncached_usd":0.003305,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7626499590696767}
{"item_id":"semeval2015.d002.s019.t003","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"},{"index":2,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"},{"index":3,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"},{"index":4,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"worker%1:18:00::","call_ids":["semeval2015.d002.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":189,"reasoning_output_tokens":173},"cost":{"currency":"USD","total_usd":0.008065000000000001,"input_uncached_usd":0.002395,"input_cached_usd":0.0,"output_usd":0.0056700000000000006,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.2090755419339985}
{"item_id":"semeval2015.d002.s019.t004","gold_sense_keys":["relate%2:42:00::","relate%2:42:01::"],"candidates":[{"index":1,"sense_key":"relate%2:31:00::","synset_id":"associate.v.01"},{"index":2,"sense_key":"relate%2:32:00::","synset_id":"relate.v.03"},{"index":3,"sense_key":"relate%2:42:01::","synset_id":"relate.v.04"},{"index":4,"sense_key":"relate%2:42:00::","synset_id":"refer.v.02"},{"index":5,"sense_key":"relate%2:41:00::","synset_id":"relate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"relate%2:42:01::","call_ids":["semeval2015.d002.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"relate%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018520000000000002,"input_uncached_usd":0.00268,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.761003624997102}
{"item_id":"semeval2015.d002.s019.t005","gold_sense_keys":["age%1:07:00::"],"candidates":[{"index":1,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"},{"index":2,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":3,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"},{"index":4,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":5,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"age%1:28:00::","call_ids":["semeval2015.d002.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"age%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01851,"input_uncached_usd":0.00267,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.132975541986525}
{"item_id":"semeval2015.d002.s019.t006","gold_sense_keys":["discrimination%1:04:00::"],"candidates":[{"index":1,"sense_key":"discrimination%1:04:00::","synset_id":"discrimination.n.01"},{"index":2,"sense_key":"discrimination%1:09:00::","synset_id":"discrimination.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discrimination%1:04:00::","call_ids":["semeval2015.d002.s019.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discrimination%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.003715,"input_uncached_usd":0.0019450000000000001,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7070805000839755}
{"item_id":"semeval2015.d002.s019.t007","gold_sense_keys":["examine%2:31:00::"],"candidates":[{"index":1,"sense_key":"examine%2:31:00::","synset_id":"analyze.v.01"},{"index":2,"sense_key":"examine%2:32:01::","synset_id":"probe.v.01"},{"index":3,"sense_key":"examine%2:32:00::","synset_id":"examine.v.04"},{"index":4,"sense_key":"examine%2:39:00::","synset_id":"examine.v.02"},{"index":5,"sense_key":"examine%2:41:00::","synset_id":"test.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"examine%2:31:00::","call_ids":["semeval2015.d002.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"examine%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":541,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.004835000000000001,"input_uncached_usd":0.0027050000000000004,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5922299579251558}
{"item_id":"semeval2015.d002.s020.t002","gold_sense_keys":["consider%2:39:00::"],"candidates":[{"index":1,"sense_key":"consider%2:32:00::","synset_id":"consider.v.04"},{"index":2,"sense_key":"consider%2:39:00::","synset_id":"study.v.03"},{"index":3,"sense_key":"consider%2:31:00::","synset_id":"see.v.05"},{"index":4,"sense_key":"consider%2:39:08::","synset_id":"view.v.02"},{"index":5,"sense_key":"consider%2:31:05::","synset_id":"consider.v.09"},{"index":6,"sense_key":"consider%2:31:08::","synset_id":"think.v.01"},{"index":7,"sense_key":"consider%2:31:01::","synset_id":"consider.v.03"},{"index":8,"sense_key":"consider%2:32:02::","synset_id":"consider.v.05"},{"index":9,"sense_key":"consider%2:39:04::","synset_id":"regard.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"consider%2:39:00::","call_ids":["semeval2015.d002.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"consider%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":1702,"reasoning_output_tokens":1668},"cost":{"currency":"USD","total_usd":0.054465,"input_uncached_usd":0.0034050000000000005,"input_cached_usd":0.0,"output_usd":0.05106,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":31.398210042039864}
{"item_id":"semeval2015.d002.s020.t004","gold_sense_keys":["age%1:07:00::"],"candidates":[{"index":1,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"},{"index":2,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":3,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"},{"index":4,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":5,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"age%1:28:00::","call_ids":["semeval2015.d002.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"age%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01848,"input_uncached_usd":0.0026400000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.22463254199829}
{"item_id":"semeval2015.d002.s020.t005","gold_sense_keys":["discrimination%1:04:00::"],"candidates":[{"index":1,"sense_key":"discrimination%1:04:00::","synset_id":"discrimination.n.01"},{"index":2,"sense_key":"discrimination%1:09:00::","synset_id":"discrimination.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discrimination%1:04:00::","call_ids":["semeval2015.d002.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discrimination%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.003235,"input_uncached_usd":0.0019150000000000003,"input_cached_usd":0.0,"output_usd":0.00132,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.667432833928615}
{"item_id":"semeval2015.d002.s020.t007","gold_sense_keys":["small%5:00:00:limited:00"],"candidates":[{"index":1,"sense_key":"small%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":2,"sense_key":"small%5:00:00:soft:04","synset_id":"little.s.03"},{"index":3,"sense_key":"small%5:00:00:moderate:00","synset_id":"modest.s.01"},{"index":4,"sense_key":"small%3:00:00::","synset_id":"small.a.01"},{"index":5,"sense_key":"small%5:00:00:decreased:00","synset_id":"belittled.s.01"},{"index":6,"sense_key":"small%5:00:00:little:03","synset_id":"small.s.03"},{"index":7,"sense_key":"small%5:00:00:limited:00","synset_id":"minor.s.03"},{"index":8,"sense_key":"small%5:00:00:young:00","synset_id":"little.s.01"},{"index":9,"sense_key":"small%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":10,"sense_key":"small%5:00:00:fine:00","synset_id":"small.s.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"small%5:00:00:limited:00","call_ids":["semeval2015.d002.s020.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"small%5:00:00:limited:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":745,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.007235,"input_uncached_usd":0.0037250000000000004,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4897341249743477}
{"item_id":"semeval2015.d002.s020.t008","gold_sense_keys":["enterprise%1:14:00::"],"candidates":[{"index":1,"sense_key":"enterprise%1:14:00::","synset_id":"enterprise.n.02"},{"index":2,"sense_key":"enterprise%1:07:00::","synset_id":"enterprise.n.03"},{"index":3,"sense_key":"enterprise%1:04:00::","synset_id":"enterprise.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"enterprise%1:14:00::","call_ids":["semeval2015.d002.s020.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"enterprise%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.0037500000000000003,"input_uncached_usd":0.0021300000000000004,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.433542083017528}
{"item_id":"semeval2015.d002.s021.t000","gold_sense_keys":["member%1:14:00::"],"candidates":[{"index":1,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":5,"sense_key":"member%1:14:00::","synset_id":"member.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"member%1:14:00::","call_ids":["semeval2015.d002.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"member%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":79},"cost":{"currency":"USD","total_usd":0.005535,"input_uncached_usd":0.0026850000000000003,"input_cached_usd":0.0,"output_usd":0.00285,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.4654417079873383}
{"item_id":"semeval2015.d002.s021.t002","gold_sense_keys":["cover%2:42:02::"],"candidates":[{"index":1,"sense_key":"cover%2:32:00::","synset_id":"cover.v.14"},{"index":2,"sense_key":"cover%2:35:14::","synset_id":"overlay.v.01"},{"index":3,"sense_key":"cover%2:29:02::","synset_id":"brood.v.05"},{"index":4,"sense_key":"cover%2:41:08::","synset_id":"cover.v.16"},{"index":5,"sense_key":"cover%2:32:12::","synset_id":"cover.v.13"},{"index":6,"sense_key":"cover%2:39:14::","synset_id":"cover.v.19"},{"index":7,"sense_key":"cover%2:33:07::","synset_id":"cover.v.23"},{"index":8,"sense_key":"cover%2:32:01::","synset_id":"cover.v.05"},{"index":9,"sense_key":"cover%2:41:01::","synset_id":"cover.v.10"},{"index":10,"sense_key":"cover%2:42:00::","synset_id":"cover.v.04"},{"index":11,"sense_key":"cover%2:42:01::","synset_id":"cover.v.03"},{"index":12,"sense_key":"cover%2:38:00::","synset_id":"traverse.v.01"},{"index":13,"sense_key":"cover%2:35:02::","synset_id":"breed.v.02"},{"index":14,"sense_key":"cover%2:39:12::","synset_id":"cover.v.11"},{"index":15,"sense_key":"cover%2:33:00::","synset_id":"cover.v.12"},{"index":16,"sense_key":"cover%2:35:00::","synset_id":"cover.v.01"},{"index":17,"sense_key":"cover%2:32:02::","synset_id":"report.v.05"},{"index":18,"sense_key":"cover%2:35:13::","synset_id":"shroud.v.01"},{"index":19,"sense_key":"cover%2:33:01::","synset_id":"cover.v.09"},{"index":20,"sense_key":"cover%2:42:02::","synset_id":"embrace.v.01"},{"index":21,"sense_key":"cover%2:33:13::","synset_id":"cover.v.24"},{"index":22,"sense_key":"cover%2:40:00::","synset_id":"cover.v.18"},{"index":23,"sense_key":"cover%2:35:01::","synset_id":"cover.v.02"},{"index":24,"sense_key":"cover%2:29:15::","synset_id":"cover.v.26"},{"index":25,"sense_key":"cover%2:42:14::","synset_id":"cover.v.15"},{"index":26,"sense_key":"cover%2:41:10::","synset_id":"cover.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":20,"chosen_sense_key":"cover%2:42:02::","call_ids":["semeval2015.d002.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":20,"predicted_sense_key":"cover%2:42:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1356,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":86},"cost":{"currency":"USD","total_usd":0.009810000000000001,"input_uncached_usd":0.0067800000000000004,"input_cached_usd":0.0,"output_usd":0.00303,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7437811660347506}
{"item_id":"semeval2015.d003.s000.t000","gold_sense_keys":["document%1:10:00::"],"candidates":[{"index":1,"sense_key":"document%1:10:01::","synset_id":"text_file.n.01"},{"index":2,"sense_key":"document%1:10:00::","synset_id":"document.n.01"},{"index":3,"sense_key":"document%1:21:00::","synset_id":"document.n.03"},{"index":4,"sense_key":"document%1:06:00::","synset_id":"document.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"document%1:10:00::","call_ids":["semeval2015.d003.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"document%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":319,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.003395,"input_uncached_usd":0.001595,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.446641792077571}
{"item_id":"semeval2015.d003.s000.t003","gold_sense_keys":["public%3:00:00::"],"candidates":[{"index":1,"sense_key":"public%5:00:00:common:02","synset_id":"public.s.01"},{"index":2,"sense_key":"public%3:00:00::","synset_id":"public.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"public%3:00:00::","call_ids":["semeval2015.d003.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"public%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":268,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":115},"cost":{"currency":"USD","total_usd":0.00527,"input_uncached_usd":0.00134,"input_cached_usd":0.0,"output_usd":0.00393,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.8102507919538766}
{"item_id":"semeval2015.d003.s000.t004","gold_sense_keys":["assessment%1:04:00::"],"candidates":[{"index":1,"sense_key":"assessment%1:04:00::","synset_id":"judgment.n.02"},{"index":2,"sense_key":"assessment%1:07:00::","synset_id":"assessment.n.03"},{"index":3,"sense_key":"assessment%1:09:00::","synset_id":"appraisal.n.01"},{"index":4,"sense_key":"assessment%1:21:00::","synset_id":"assessment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assessment%1:04:00::","call_ids":["semeval2015.d003.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assessment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":329,"cached_input_tokens":0,"output_tokens":223,"reasoning_output_tokens":207},"cost":{"currency":"USD","total_usd":0.008335,"input_uncached_usd":0.0016450000000000002,"input_cached_usd":0.0,"output_usd":0.00669,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.225390249979682}
{"item_id":"semeval2015.d003.s000.t005","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":2,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":5,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":6,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"},{"index":7,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2015.d003.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.004410000000000001,"input_uncached_usd":0.0025800000000000003,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6329375830246136}
{"item_id":"semeval2015.d003.s001.t000","gold_sense_keys":["explain%2:32:00::","explain%2:32:01::"],"candidates":[{"index":1,"sense_key":"explain%2:32:01::","synset_id":"explain.v.02"},{"index":2,"sense_key":"explain%2:32:02::","synset_id":"excuse.v.03"},{"index":3,"sense_key":"explain%2:32:00::","synset_id":"explain.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"explain%2:32:00::","call_ids":["semeval2015.d003.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"explain%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":333,"cached_input_tokens":0,"output_tokens":263,"reasoning_output_tokens":247},"cost":{"currency":"USD","total_usd":0.009555,"input_uncached_usd":0.001665,"input_cached_usd":0.0,"output_usd":0.00789,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.325369292055257}
{"item_id":"semeval2015.d003.s001.t001","gold_sense_keys":["committee%1:14:00::"],"candidates":[{"index":1,"sense_key":"committee%1:14:00::","synset_id":"committee.n.01"},{"index":2,"sense_key":"committee%1:14:01::","synset_id":"committee.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"committee%1:14:00::","call_ids":["semeval2015.d003.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"committee%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":276,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.0034500000000000004,"input_uncached_usd":0.0013800000000000002,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8200431659352034}
{"item_id":"semeval2015.d003.s001.t003","gold_sense_keys":["product%1:06:01::"],"candidates":[{"index":1,"sense_key":"product%1:06:00::","synset_id":"product.n.02"},{"index":2,"sense_key":"product%1:06:01::","synset_id":"merchandise.n.01"},{"index":3,"sense_key":"product%1:14:00::","synset_id":"intersection.n.04"},{"index":4,"sense_key":"product%1:27:00::","synset_id":"product.n.04"},{"index":5,"sense_key":"product%1:19:00::","synset_id":"product.n.05"},{"index":6,"sense_key":"product%1:09:00::","synset_id":"product.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"product%1:06:01::","call_ids":["semeval2015.d003.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"product%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01823,"input_uncached_usd":0.00239,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.197645416948944}
{"item_id":"semeval2015.d003.s001.t005","gold_sense_keys":["use%1:04:00::"],"candidates":[{"index":1,"sense_key":"use%1:04:01::","synset_id":"habit.n.02"},{"index":2,"sense_key":"use%1:07:02::","synset_id":"use.n.03"},{"index":3,"sense_key":"use%1:22:00::","synset_id":"consumption.n.03"},{"index":4,"sense_key":"use%1:04:00::","synset_id":"use.n.01"},{"index":5,"sense_key":"use%1:07:03::","synset_id":"use.n.07"},{"index":6,"sense_key":"use%1:07:00::","synset_id":"function.n.02"},{"index":7,"sense_key":"use%1:04:02::","synset_id":"manipulation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"use%1:04:00::","call_ids":["semeval2015.d003.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"use%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018615,"input_uncached_usd":0.002775,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.525384124950506}
{"item_id":"semeval2015.d003.s001.t006","gold_sense_keys":["assess%2:31:00::"],"candidates":[{"index":1,"sense_key":"assess%2:40:01::","synset_id":"assess.v.02"},{"index":2,"sense_key":"assess%2:31:00::","synset_id":"measure.v.04"},{"index":3,"sense_key":"assess%2:31:01::","synset_id":"assess.v.04"},{"index":4,"sense_key":"assess%2:40:00::","synset_id":"tax.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"assess%2:31:00::","call_ids":["semeval2015.d003.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"assess%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":378,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.0036000000000000003,"input_uncached_usd":0.0018900000000000002,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4220389999682084}
{"item_id":"semeval2015.d003.s001.t007","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":2,"sense_key":"study%1:10:01::","synset_id":"study.n.10"},{"index":3,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":4,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":5,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":6,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":7,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":8,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":9,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":10,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d003.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":372,"reasoning_output_tokens":356},"cost":{"currency":"USD","total_usd":0.014225,"input_uncached_usd":0.0030650000000000004,"input_cached_usd":0.0,"output_usd":0.01116,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.066695874906145}
{"item_id":"semeval2015.d003.s001.t008","gold_sense_keys":["perform%2:36:00::"],"candidates":[{"index":1,"sense_key":"perform%2:41:00::","synset_id":"do.v.03"},{"index":2,"sense_key":"perform%2:41:02::","synset_id":"perform.v.02"},{"index":3,"sense_key":"perform%2:36:01::","synset_id":"perform.v.03"},{"index":4,"sense_key":"perform%2:36:00::","synset_id":"perform.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"perform%2:36:00::","call_ids":["semeval2015.d003.s001.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"perform%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":358,"cached_input_tokens":0,"output_tokens":166,"reasoning_output_tokens":151},"cost":{"currency":"USD","total_usd":0.00677,"input_uncached_usd":0.0017900000000000001,"input_cached_usd":0.0,"output_usd":0.00498,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.233574250014499}
{"item_id":"semeval2015.d003.s001.t009","gold_sense_keys":["reach%2:38:01::","reach%2:38:06::","reach%2:41:00::"],"candidates":[{"index":1,"sense_key":"reach%2:33:00::","synset_id":"strive.v.02"},{"index":2,"sense_key":"reach%2:41:00::","synset_id":"achieve.v.01"},{"index":3,"sense_key":"reach%2:40:00::","synset_id":"pass.v.05"},{"index":4,"sense_key":"reach%2:32:00::","synset_id":"reach.v.04"},{"index":5,"sense_key":"reach%2:38:01::","synset_id":"reach.v.01"},{"index":6,"sense_key":"reach%2:38:00::","synset_id":"reach.v.02"},{"index":7,"sense_key":"reach%2:38:06::","synset_id":"reach.v.07"},{"index":8,"sense_key":"reach%2:35:00::","synset_id":"reach.v.03"},{"index":9,"sense_key":"reach%2:42:00::","synset_id":"reach.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"reach%2:38:01::","call_ids":["semeval2015.d003.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"reach%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":279,"reasoning_output_tokens":263},"cost":{"currency":"USD","total_usd":0.011484999999999999,"input_uncached_usd":0.003115,"input_cached_usd":0.0,"output_usd":0.00837,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.667785124969669}
{"item_id":"semeval2015.d003.s001.t010","gold_sense_keys":["recommendation%1:10:00::"],"candidates":[{"index":1,"sense_key":"recommendation%1:10:02::","synset_id":"recommendation.n.02"},{"index":2,"sense_key":"recommendation%1:07:00::","synset_id":"recommendation.n.03"},{"index":3,"sense_key":"recommendation%1:10:00::","synset_id":"recommendation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"recommendation%1:10:00::","call_ids":["semeval2015.d003.s001.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"recommendation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":333,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.0032849999999999997,"input_uncached_usd":0.001665,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6373142920201644}
{"item_id":"semeval2015.d003.s001.t011","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":2,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":3,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":4,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":5,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":6,"sense_key":"use%2:34:02::","synset_id":"use.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d003.s001.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":155,"reasoning_output_tokens":140},"cost":{"currency":"USD","total_usd":0.00705,"input_uncached_usd":0.0024000000000000002,"input_cached_usd":0.0,"output_usd":0.0046500000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.4448590830434114}
{"item_id":"semeval2015.d003.s001.t012","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"},{"index":4,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d003.s001.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":61},"cost":{"currency":"USD","total_usd":0.00413,"input_uncached_usd":0.00185,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9904426670400426}
{"item_id":"semeval2015.d003.s003.t000","gold_sense_keys":["contain%2:42:00::"],"candidates":[{"index":1,"sense_key":"contain%2:33:00::","synset_id":"check.v.18"},{"index":2,"sense_key":"contain%2:42:00::","synset_id":"incorporate.v.02"},{"index":3,"sense_key":"contain%2:42:06::","synset_id":"contain.v.04"},{"index":4,"sense_key":"contain%2:42:13::","synset_id":"hold.v.11"},{"index":5,"sense_key":"contain%2:41:00::","synset_id":"control.v.02"},{"index":6,"sense_key":"contain%2:42:14::","synset_id":"contain.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"contain%2:42:00::","call_ids":["semeval2015.d003.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"contain%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":542,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.00571,"input_uncached_usd":0.00271,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.038041624939069}
{"item_id":"semeval2015.d003.s003.t001","gold_sense_keys":["active%3:00:07::"],"candidates":[{"index":1,"sense_key":"active%3:00:04::","synset_id":"active.a.13"},{"index":2,"sense_key":"active%3:00:08::","synset_id":"active.a.14"},{"index":3,"sense_key":"active%3:00:05::","synset_id":"active.a.12"},{"index":4,"sense_key":"active%3:00:01::","synset_id":"active.a.05"},{"index":5,"sense_key":"active%3:00:03::","synset_id":"active.a.03"},{"index":6,"sense_key":"active%3:00:06::","synset_id":"active.a.07"},{"index":7,"sense_key":"active%3:00:10::","synset_id":"active.a.09"},{"index":8,"sense_key":"active%5:00:00:involved:00","synset_id":"active.s.02"},{"index":9,"sense_key":"active%3:00:09::","synset_id":"active.a.10"},{"index":10,"sense_key":"active%5:00:00:existent:00","synset_id":"active.s.03"},{"index":11,"sense_key":"active%3:00:02::","synset_id":"active.a.01"},{"index":12,"sense_key":"active%3:00:07::","synset_id":"active.a.06"},{"index":13,"sense_key":"active%3:00:00::","synset_id":"active.a.11"},{"index":14,"sense_key":"active%5:00:00:operational:00","synset_id":"active.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"active%3:00:07::","call_ids":["semeval2015.d003.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"active%3:00:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":786,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.0058200000000000005,"input_uncached_usd":0.00393,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.840547417057678}
{"item_id":"semeval2015.d003.s003.t002","gold_sense_keys":["substance%1:03:01::"],"candidates":[{"index":1,"sense_key":"substance%1:03:00::","synset_id":"substance.n.01"},{"index":2,"sense_key":"substance%1:09:01::","synset_id":"kernel.n.03"},{"index":3,"sense_key":"substance%1:21:00::","synset_id":"means.n.03"},{"index":4,"sense_key":"substance%1:03:01::","synset_id":"substance.n.07"},{"index":5,"sense_key":"substance%1:09:00::","synset_id":"meaning.n.02"},{"index":6,"sense_key":"substance%1:10:00::","synset_id":"message.n.02"},{"index":7,"sense_key":"substance%1:27:00::","synset_id":"substance.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"substance%1:03:01::","call_ids":["semeval2015.d003.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"substance%1:03:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":328,"reasoning_output_tokens":313},"cost":{"currency":"USD","total_usd":0.012485000000000001,"input_uncached_usd":0.002645,"input_cached_usd":0.0,"output_usd":0.00984,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.830780292046256}
{"item_id":"semeval2015.d003.s003.t003","gold_sense_keys":["available%3:00:00::"],"candidates":[{"index":1,"sense_key":"available%5:00:00:free:00","synset_id":"available.s.01"},{"index":2,"sense_key":"available%3:00:00::","synset_id":"available.a.01"},{"index":3,"sense_key":"available%5:00:00:disposable:02","synset_id":"available.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"available%3:00:00::","call_ids":["semeval2015.d003.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"available%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":392,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.00421,"input_uncached_usd":0.0019600000000000004,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7590944579569623}
{"item_id":"semeval2015.d003.s003.t004","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"},{"index":2,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"},{"index":3,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"},{"index":4,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0037400000000000003,"input_uncached_usd":0.00185,"input_cached_usd":0.0,"output_usd":0.00189,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4108459999551997}
{"item_id":"semeval2015.d003.s003.t005","gold_sense_keys":["solution%1:27:00::"],"candidates":[{"index":1,"sense_key":"solution%1:09:00::","synset_id":"solution.n.03"},{"index":2,"sense_key":"solution%1:27:00::","synset_id":"solution.n.01"},{"index":3,"sense_key":"solution%1:04:00::","synset_id":"solution.n.05"},{"index":4,"sense_key":"solution%1:10:00::","synset_id":"solution.n.02"},{"index":5,"sense_key":"solution%1:14:00::","synset_id":"solution.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"solution%1:27:00::","call_ids":["semeval2015.d003.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"solution%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.003885,"input_uncached_usd":0.002235,"input_cached_usd":0.0,"output_usd":0.00165,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.9684087089262903}
{"item_id":"semeval2015.d003.s003.t006","gold_sense_keys":["injection%1:04:01::"],"candidates":[{"index":1,"sense_key":"injection%1:27:00::","synset_id":"injection.n.02"},{"index":2,"sense_key":"injection%1:04:00::","synset_id":"injection.n.01"},{"index":3,"sense_key":"injection%1:04:01::","synset_id":"injection.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"injection%1:04:01::","call_ids":["semeval2015.d003.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"injection%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":344,"cached_input_tokens":0,"output_tokens":346,"reasoning_output_tokens":330},"cost":{"currency":"USD","total_usd":0.0121,"input_uncached_usd":0.0017200000000000002,"input_cached_usd":0.0,"output_usd":0.01038,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.977160583017394}
{"item_id":"semeval2015.d003.s004.t000","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"},{"index":2,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"},{"index":3,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"},{"index":4,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":29},"cost":{"currency":"USD","total_usd":0.00332,"input_uncached_usd":0.002,"input_cached_usd":0.0,"output_usd":0.00132,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6289921249262989}
{"item_id":"semeval2015.d003.s004.t001","gold_sense_keys":["have%2:40:00::","have%2:42:00::"],"candidates":[{"index":1,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":2,"sense_key":"have%2:35:00::","synset_id":"take.v.35"},{"index":3,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":4,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":5,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":6,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":7,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":10,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":11,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":12,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":13,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":14,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":15,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":18,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":19,"sense_key":"have%2:30:01::","synset_id":"have.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"have%2:42:00::","call_ids":["semeval2015.d003.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"have%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1105,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":81},"cost":{"currency":"USD","total_usd":0.008405,"input_uncached_usd":0.005525,"input_cached_usd":0.0,"output_usd":0.00288,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.059067416936159}
{"item_id":"semeval2015.d003.s004.t002","gold_sense_keys":["score%1:25:00::"],"candidates":[{"index":1,"sense_key":"score%1:04:01::","synset_id":"sexual_conquest.n.01"},{"index":2,"sense_key":"score%1:26:00::","synset_id":"score.n.07"},{"index":3,"sense_key":"score%1:10:00::","synset_id":"score.n.02"},{"index":4,"sense_key":"score%1:04:00::","synset_id":"score.n.10"},{"index":5,"sense_key":"score%1:16:00::","synset_id":"score.n.05"},{"index":6,"sense_key":"score%1:23:00::","synset_id":"score.n.03"},{"index":7,"sense_key":"score%1:25:00::","synset_id":"score.n.08"},{"index":8,"sense_key":"score%1:09:00::","synset_id":"mark.n.01"},{"index":9,"sense_key":"score%1:14:00::","synset_id":"score.n.04"},{"index":10,"sense_key":"score%1:12:00::","synset_id":"grudge.n.01"},{"index":11,"sense_key":"score%1:09:01::","synset_id":"score.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"score%1:25:00::","call_ids":["semeval2015.d003.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"score%1:25:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":77},"cost":{"currency":"USD","total_usd":0.0064150000000000006,"input_uncached_usd":0.003625,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.850842332933098}
{"item_id":"semeval2015.d003.s004.t003","gold_sense_keys":["line%1:25:01::"],"candidates":[{"index":1,"sense_key":"line%1:06:09::","synset_id":"production_line.n.01"},{"index":2,"sense_key":"line%1:06:03::","synset_id":"line.n.14"},{"index":3,"sense_key":"line%1:10:03::","synset_id":"tune.n.01"},{"index":4,"sense_key":"line%1:04:01::","synset_id":"line.n.16"},{"index":5,"sense_key":"line%1:10:02::","synset_id":"line.n.05"},{"index":6,"sense_key":"line%1:10:06::","synset_id":"line.n.27"},{"index":7,"sense_key":"line%1:14:01::","synset_id":"line.n.03"},{"index":8,"sense_key":"line%1:10:04::","synset_id":"note.n.02"},{"index":9,"sense_key":"line%1:25:01::","synset_id":"wrinkle.n.01"},{"index":10,"sense_key":"line%1:09:01::","synset_id":"argumentation.n.02"},{"index":11,"sense_key":"line%1:21:00::","synset_id":"credit_line.n.01"},{"index":12,"sense_key":"line%1:06:01::","synset_id":"cable.n.02"},{"index":13,"sense_key":"line%1:15:01::","synset_id":"line.n.20"},{"index":14,"sense_key":"line%1:04:00::","synset_id":"occupation.n.01"},{"index":15,"sense_key":"line%1:15:02::","synset_id":"line.n.07"},{"index":16,"sense_key":"line%1:06:05::","synset_id":"line.n.23"},{"index":17,"sense_key":"line%1:15:00::","synset_id":"line.n.11"},{"index":18,"sense_key":"line%1:14:00::","synset_id":"lineage.n.01"},{"index":19,"sense_key":"line%1:23:00::","synset_id":"agate_line.n.01"},{"index":20,"sense_key":"line%1:19:00::","synset_id":"line.n.06"},{"index":21,"sense_key":"line%1:10:05::","synset_id":"channel.n.05"},{"index":22,"sense_key":"line%1:10:01::","synset_id":"line.n.02"},{"index":23,"sense_key":"line%1:14:02::","synset_id":"course.n.02"},{"index":24,"sense_key":"line%1:25:00::","synset_id":"line.n.04"},{"index":25,"sense_key":"line%1:06:07::","synset_id":"telephone_line.n.02"},{"index":26,"sense_key":"line%1:09:00::","synset_id":"line.n.29"},{"index":27,"sense_key":"line%1:14:03::","synset_id":"line.n.01"},{"index":28,"sense_key":"line%1:06:00::","synset_id":"line.n.18"},{"index":29,"sense_key":"line%1:06:06::","synset_id":"pipeline.n.02"},{"index":30,"sense_key":"line%1:06:08::","synset_id":"line.n.22"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"line%1:25:01::","call_ids":["semeval2015.d003.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"line%1:25:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1516,"cached_input_tokens":0,"output_tokens":277,"reasoning_output_tokens":261},"cost":{"currency":"USD","total_usd":0.01589,"input_uncached_usd":0.007580000000000001,"input_cached_usd":0.0,"output_usd":0.00831,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.384914333000779}
{"item_id":"semeval2015.d003.s004.t004","gold_sense_keys":["side%1:06:00::","side%1:15:00::"],"candidates":[{"index":1,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":2,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":3,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":4,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":5,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":6,"sense_key":"side%1:11:00::","synset_id":"english.n.04"},{"index":7,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":8,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":9,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":10,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":11,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":12,"sense_key":"side%1:14:00::","synset_id":"side.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"side%1:15:00::","call_ids":["semeval2015.d003.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"side%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":778,"cached_input_tokens":0,"output_tokens":759,"reasoning_output_tokens":737},"cost":{"currency":"USD","total_usd":0.026660000000000003,"input_uncached_usd":0.0038900000000000002,"input_cached_usd":0.0,"output_usd":0.022770000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.361561291967519}
{"item_id":"semeval2015.d003.s004.t005","gold_sense_keys":["so%4:02:06::"],"candidates":[{"index":1,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":2,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":3,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":4,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"},{"index":5,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":6,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":7,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":8,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":9,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":10,"sense_key":"so%4:02:05::","synset_id":"so.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"so%4:02:06::","call_ids":["semeval2015.d003.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"so%4:02:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":752,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":103},"cost":{"currency":"USD","total_usd":0.00733,"input_uncached_usd":0.0037600000000000003,"input_cached_usd":0.0,"output_usd":0.0035700000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.3486768340226263}
{"item_id":"semeval2015.d003.s005.t000","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"},{"index":2,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"},{"index":3,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"},{"index":4,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0037150000000000004,"input_uncached_usd":0.0020350000000000004,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7013668330619112}
{"item_id":"semeval2015.d003.s005.t001","gold_sense_keys":["marked%3:00:00::"],"candidates":[{"index":1,"sense_key":"marked%5:00:00:conspicuous:00","synset_id":"marked.s.02"},{"index":2,"sense_key":"marked%5:00:00:noticeable:00","synset_id":"marked.s.01"},{"index":3,"sense_key":"marked%3:00:00::","synset_id":"marked.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"marked%3:00:00::","call_ids":["semeval2015.d003.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"marked%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":33},"cost":{"currency":"USD","total_usd":0.00352,"input_uncached_usd":0.00205,"input_cached_usd":0.0,"output_usd":0.00147,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4315376250306144}
{"item_id":"semeval2015.d003.s005.t003","gold_sense_keys":["side%1:06:00::","side%1:15:00::"],"candidates":[{"index":1,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":2,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":3,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":4,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":5,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":6,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":7,"sense_key":"side%1:11:00::","synset_id":"english.n.04"},{"index":8,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":9,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":10,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":11,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":12,"sense_key":"side%1:10:00::","synset_id":"side.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"side%1:15:00::","call_ids":["semeval2015.d003.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"side%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":786,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01977,"input_uncached_usd":0.00393,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.734404082992114}
{"item_id":"semeval2015.d003.s005.t004","gold_sense_keys":["letter%1:10:01::"],"candidates":[{"index":1,"sense_key":"letter%1:18:00::","synset_id":"letter.n.03"},{"index":2,"sense_key":"letter%1:10:00::","synset_id":"letter.n.01"},{"index":3,"sense_key":"letter%1:10:03::","synset_id":"letter.n.04"},{"index":4,"sense_key":"letter%1:10:01::","synset_id":"letter.n.02"},{"index":5,"sense_key":"letter%1:10:02::","synset_id":"letter.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"letter%1:10:01::","call_ids":["semeval2015.d003.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"letter%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":45},"cost":{"currency":"USD","total_usd":0.00421,"input_uncached_usd":0.0024100000000000002,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6027758750133216}
{"item_id":"semeval2015.d003.s005.t005","gold_sense_keys":["quantity%1:03:00::"],"candidates":[{"index":1,"sense_key":"quantity%1:07:00::","synset_id":"quantity.n.02"},{"index":2,"sense_key":"quantity%1:03:00::","synset_id":"measure.n.02"},{"index":3,"sense_key":"quantity%1:09:01::","synset_id":"quantity.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"quantity%1:03:00::","call_ids":["semeval2015.d003.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"quantity%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.0047350000000000005,"input_uncached_usd":0.0019150000000000003,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.1743659999920055}
{"item_id":"semeval2015.d003.s005.t006","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":4,"sense_key":"other%3:00:00::","synset_id":"other.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"other%3:00:00::","call_ids":["semeval2015.d003.s005.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.004205,"input_uncached_usd":0.002255,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0655162499751896}
{"item_id":"semeval2015.d003.s005.t007","gold_sense_keys":["side%1:06:00::","side%1:15:00::"],"candidates":[{"index":1,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":2,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":3,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":4,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":5,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":6,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":7,"sense_key":"side%1:11:00::","synset_id":"english.n.04"},{"index":8,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":9,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":10,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":11,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":12,"sense_key":"side%1:15:02::","synset_id":"side.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"side%1:15:00::","call_ids":["semeval2015.d003.s005.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"side%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":785,"cached_input_tokens":0,"output_tokens":915,"reasoning_output_tokens":893},"cost":{"currency":"USD","total_usd":0.031375,"input_uncached_usd":0.0039250000000000005,"input_cached_usd":0.0,"output_usd":0.027450000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":13.028185290982947}
{"item_id":"semeval2015.d003.s006.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":2,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":5,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":6,"sense_key":"use%2:34:01::","synset_id":"use.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d003.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.005005000000000001,"input_uncached_usd":0.0028150000000000002,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.517323874984868}
{"item_id":"semeval2015.d003.s007.t001","gold_sense_keys":["mean%2:32:03::","mean%2:42:00::"],"candidates":[{"index":1,"sense_key":"mean%2:32:03::","synset_id":"mean.v.03"},{"index":2,"sense_key":"mean%2:42:00::","synset_id":"entail.v.01"},{"index":3,"sense_key":"mean%2:31:00::","synset_id":"intend.v.01"},{"index":4,"sense_key":"mean%2:42:03::","synset_id":"mean.v.05"},{"index":5,"sense_key":"mean%2:31:05::","synset_id":"mean.v.07"},{"index":6,"sense_key":"mean%2:32:01::","synset_id":"mean.v.01"},{"index":7,"sense_key":"mean%2:31:01::","synset_id":"think_of.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mean%2:42:00::","call_ids":["semeval2015.d003.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mean%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018815,"input_uncached_usd":0.002975,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.966109666973352}
{"item_id":"semeval2015.d003.s007.t002","gold_sense_keys":["stop%2:41:00::"],"candidates":[{"index":1,"sense_key":"stop%2:35:00::","synset_id":"intercept.v.01"},{"index":2,"sense_key":"stop%2:38:00::","synset_id":"stop.v.01"},{"index":3,"sense_key":"stop%2:38:01::","synset_id":"stop.v.05"},{"index":4,"sense_key":"stop%2:38:02::","synset_id":"stop.v.04"},{"index":5,"sense_key":"stop%2:42:13::","synset_id":"end.v.01"},{"index":6,"sense_key":"stop%2:42:00::","synset_id":"discontinue.v.01"},{"index":7,"sense_key":"stop%2:30:12::","synset_id":"hold_on.v.02"},{"index":8,"sense_key":"stop%2:33:00::","synset_id":"check.v.18"},{"index":9,"sense_key":"stop%2:41:00::","synset_id":"stop.v.03"},{"index":10,"sense_key":"stop%2:35:12::","synset_id":"barricade.v.01"},{"index":11,"sense_key":"stop%2:30:05::","synset_id":"break.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"stop%2:41:00::","call_ids":["semeval2015.d003.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"stop%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":802,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01985,"input_uncached_usd":0.0040100000000000005,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.320931834052317}
{"item_id":"semeval2015.d003.s008.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":2,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":3,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":6,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d003.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":40},"cost":{"currency":"USD","total_usd":0.004665000000000001,"input_uncached_usd":0.0029850000000000002,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5401245000539348}
{"item_id":"semeval2015.d003.s008.t001","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":4,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":5,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":6,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"dog%1:05:00::","call_ids":["semeval2015.d003.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.00486,"input_uncached_usd":0.0032700000000000003,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6062497090315446}
{"item_id":"semeval2015.d003.s008.t002","gold_sense_keys":["prevent%2:41:01::"],"candidates":[{"index":1,"sense_key":"prevent%2:41:00::","synset_id":"prevent.v.02"},{"index":2,"sense_key":"prevent%2:41:01::","synset_id":"prevent.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"prevent%2:41:01::","call_ids":["semeval2015.d003.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"prevent%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":101},"cost":{"currency":"USD","total_usd":0.00568,"input_uncached_usd":0.00217,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.188858999987133}
{"item_id":"semeval2015.d003.s008.t005","gold_sense_keys":["cause%2:36:00::"],"candidates":[{"index":1,"sense_key":"cause%2:32:00::","synset_id":"induce.v.02"},{"index":2,"sense_key":"cause%2:36:00::","synset_id":"cause.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cause%2:36:00::","call_ids":["semeval2015.d003.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cause%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.00426,"input_uncached_usd":0.0021300000000000004,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2418234170181677}
{"item_id":"semeval2015.d003.s008.t007","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":2,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":3,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"},{"index":4,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d003.s008.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":39},"cost":{"currency":"USD","total_usd":0.004055,"input_uncached_usd":0.002435,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4529115409823135}
{"item_id":"semeval2015.d003.s008.t008","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":2,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":3,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":4,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":5,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":6,"sense_key":"use%2:34:02::","synset_id":"use.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d003.s008.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.004965000000000001,"input_uncached_usd":0.0029850000000000002,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.7372086249524727}
{"item_id":"semeval2015.d003.s008.t009","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":2,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"},{"index":3,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":4,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s008.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":63},"cost":{"currency":"USD","total_usd":0.004830000000000001,"input_uncached_usd":0.0024600000000000004,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.179713833029382}
{"item_id":"semeval2015.d003.s008.t010","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":2,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":3,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":4,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"},{"index":5,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d003.s008.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":31,"reasoning_output_tokens":16},"cost":{"currency":"USD","total_usd":0.0035050000000000003,"input_uncached_usd":0.002575,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.1216540000168607}
{"item_id":"semeval2015.d003.s008.t012","gold_sense_keys":["cause%2:36:00::"],"candidates":[{"index":1,"sense_key":"cause%2:36:00::","synset_id":"cause.v.01"},{"index":2,"sense_key":"cause%2:32:00::","synset_id":"induce.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cause%2:36:00::","call_ids":["semeval2015.d003.s008.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cause%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":425,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":47},"cost":{"currency":"USD","total_usd":0.003985000000000001,"input_uncached_usd":0.002125,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.972510916995816}
{"item_id":"semeval2015.d003.s009.t002","gold_sense_keys":["measure%1:04:01::"],"candidates":[{"index":1,"sense_key":"measure%1:10:03::","synset_id":"standard.n.01"},{"index":2,"sense_key":"measure%1:10:02::","synset_id":"bill.n.01"},{"index":3,"sense_key":"measure%1:10:01::","synset_id":"meter.n.03"},{"index":4,"sense_key":"measure%1:04:01::","synset_id":"measure.n.01"},{"index":5,"sense_key":"measure%1:10:00::","synset_id":"measure.n.07"},{"index":6,"sense_key":"measure%1:06:00::","synset_id":"measuring_stick.n.01"},{"index":7,"sense_key":"measure%1:06:01::","synset_id":"measure.n.09"},{"index":8,"sense_key":"measure%1:04:00::","synset_id":"measurement.n.01"},{"index":9,"sense_key":"measure%1:03:00::","synset_id":"measure.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"measure%1:04:01::","call_ids":["semeval2015.d003.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"measure%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":69},"cost":{"currency":"USD","total_usd":0.0061,"input_uncached_usd":0.00355,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8256117919227108}
{"item_id":"semeval2015.d003.s009.t003","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":3,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":4,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["semeval2015.d003.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.00413,"input_uncached_usd":0.0024200000000000003,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.6145324170356616}
{"item_id":"semeval2015.d003.s009.t005","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"},{"index":4,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":70},"cost":{"currency":"USD","total_usd":0.004985,"input_uncached_usd":0.002435,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.763908875058405}
{"item_id":"semeval2015.d003.s009.t006","gold_sense_keys":["special%5:00:00:specialized:00","special%5:00:01:specific:00"],"candidates":[{"index":1,"sense_key":"special%5:00:00:specific:00","synset_id":"limited.s.05"},{"index":2,"sense_key":"special%5:00:00:extraordinary:00","synset_id":"special.s.02"},{"index":3,"sense_key":"special%5:00:00:unscheduled:00","synset_id":"extra.s.02"},{"index":4,"sense_key":"special%5:00:00:uncommon:00","synset_id":"especial.s.01"},{"index":5,"sense_key":"special%5:00:00:primary:00","synset_id":"particular.s.04"},{"index":6,"sense_key":"special%5:00:00:specialized:00","synset_id":"special.s.04"},{"index":7,"sense_key":"special%5:00:01:specific:00","synset_id":"particular.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"special%5:00:00:specialized:00","call_ids":["semeval2015.d003.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"special%5:00:00:specialized:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":118},"cost":{"currency":"USD","total_usd":0.007185,"input_uncached_usd":0.0031650000000000003,"input_cached_usd":0.0,"output_usd":0.00402,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.64240241702646}
{"item_id":"semeval2015.d003.s009.t007","gold_sense_keys":["diet%1:13:00::"],"candidates":[{"index":1,"sense_key":"diet%1:14:00::","synset_id":"diet.n.02"},{"index":2,"sense_key":"diet%1:13:01::","synset_id":"diet.n.03"},{"index":3,"sense_key":"diet%1:13:00::","synset_id":"diet.n.01"},{"index":4,"sense_key":"diet%1:04:00::","synset_id":"diet.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"diet%1:13:00::","call_ids":["semeval2015.d003.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"diet%1:13:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":55},"cost":{"currency":"USD","total_usd":0.00425,"input_uncached_usd":0.0021200000000000004,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9699534999672323}
{"item_id":"semeval2015.d003.s009.t009","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":2,"sense_key":"use%2:41:03::","synset_id":"use.v.06"},{"index":3,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":4,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":5,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":6,"sense_key":"use%2:41:14::","synset_id":"use.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d003.s009.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.00476,"input_uncached_usd":0.0029600000000000004,"input_cached_usd":0.0,"output_usd":0.0018,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.4299972919980064}
{"item_id":"semeval2015.d003.s009.t010","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":3,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s009.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":73},"cost":{"currency":"USD","total_usd":0.005105,"input_uncached_usd":0.002435,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.508702666964382}
{"item_id":"semeval2015.d003.s009.t012","gold_sense_keys":["injection%1:04:01::"],"candidates":[{"index":1,"sense_key":"injection%1:27:00::","synset_id":"injection.n.02"},{"index":2,"sense_key":"injection%1:04:01::","synset_id":"injection.n.03"},{"index":3,"sense_key":"injection%1:04:00::","synset_id":"injection.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"injection%1:27:00::","call_ids":["semeval2015.d003.s009.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"injection%1:27:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017915,"input_uncached_usd":0.002075,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.140705207944848}
{"item_id":"semeval2015.d003.s009.t013","gold_sense_keys":["injection%1:04:01::"],"candidates":[{"index":1,"sense_key":"injection%1:04:01::","synset_id":"injection.n.03"},{"index":2,"sense_key":"injection%1:04:00::","synset_id":"injection.n.01"},{"index":3,"sense_key":"injection%1:27:00::","synset_id":"injection.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"injection%1:04:01::","call_ids":["semeval2015.d003.s009.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"injection%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017885,"input_uncached_usd":0.002075,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.075661625014618}
{"item_id":"semeval2015.d003.s009.t015","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"},{"index":2,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"},{"index":3,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"},{"index":4,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s009.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":59},"cost":{"currency":"USD","total_usd":0.004450000000000001,"input_uncached_usd":0.0022,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9288637500721961}
{"item_id":"semeval2015.d003.s010.t000","gold_sense_keys":["purpose%1:07:00::"],"candidates":[{"index":1,"sense_key":"purpose%1:07:01::","synset_id":"determination.n.02"},{"index":2,"sense_key":"purpose%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"purpose%1:09:00::","synset_id":"purpose.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"purpose%1:07:00::","call_ids":["semeval2015.d003.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"purpose%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":201,"reasoning_output_tokens":185},"cost":{"currency":"USD","total_usd":0.008305,"input_uncached_usd":0.002275,"input_cached_usd":0.0,"output_usd":0.00603,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.8056317920563743}
{"item_id":"semeval2015.d003.s010.t001","gold_sense_keys":["give%2:40:13::"],"candidates":[{"index":1,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":2,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":3,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":4,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":5,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":6,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":7,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":8,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":9,"sense_key":"give%2:29:10::","synset_id":"give.v.44"},{"index":10,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":11,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":12,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":13,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":14,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":15,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":16,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":17,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":18,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":19,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":20,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":21,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":22,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":23,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":24,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":25,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":26,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":27,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":28,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":29,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":30,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":31,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":32,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":33,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":34,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":35,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":36,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":37,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":38,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":39,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":40,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":41,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":42,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":43,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":44,"sense_key":"give%2:36:12::","synset_id":"give.v.32"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"give%2:40:13::","call_ids":["semeval2015.d003.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"give%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2044,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.012620000000000003,"input_uncached_usd":0.010220000000000002,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8369426659774035}
{"item_id":"semeval2015.d003.s010.t002","gold_sense_keys":["day%1:28:00::"],"candidates":[{"index":1,"sense_key":"day%1:18:00::","synset_id":"day.n.10"},{"index":2,"sense_key":"day%1:28:01::","synset_id":"day.n.03"},{"index":3,"sense_key":"day%1:28:03::","synset_id":"day.n.02"},{"index":4,"sense_key":"day%1:28:02::","synset_id":"day.n.04"},{"index":5,"sense_key":"day%1:28:07::","synset_id":"day.n.07"},{"index":6,"sense_key":"day%1:28:05::","synset_id":"day.n.06"},{"index":7,"sense_key":"day%1:28:04::","synset_id":"day.n.05"},{"index":8,"sense_key":"day%1:28:00::","synset_id":"day.n.01"},{"index":9,"sense_key":"day%1:28:06::","synset_id":"sidereal_day.n.01"},{"index":10,"sense_key":"day%1:26:00::","synset_id":"day.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"day%1:28:00::","call_ids":["semeval2015.d003.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"day%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":759,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":48},"cost":{"currency":"USD","total_usd":0.0057150000000000005,"input_uncached_usd":0.003795,"input_cached_usd":0.0,"output_usd":0.00192,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8742297500139102}
{"item_id":"semeval2015.d003.s012.t001","gold_sense_keys":["receptor%1:08:01::"],"candidates":[{"index":1,"sense_key":"receptor%1:08:00::","synset_id":"sense_organ.n.01"},{"index":2,"sense_key":"receptor%1:08:01::","synset_id":"receptor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"receptor%1:08:01::","call_ids":["semeval2015.d003.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"receptor%1:08:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":42},"cost":{"currency":"USD","total_usd":0.0036550000000000003,"input_uncached_usd":0.0019150000000000003,"input_cached_usd":0.0,"output_usd":0.00174,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.076792791020125}
{"item_id":"semeval2015.d003.s013.t000","gold_sense_keys":["study%2:31:02::"],"candidates":[{"index":1,"sense_key":"study%2:31:01::","synset_id":"study.v.05"},{"index":2,"sense_key":"study%2:39:00::","synset_id":"study.v.03"},{"index":3,"sense_key":"study%2:31:04::","synset_id":"study.v.06"},{"index":4,"sense_key":"study%2:31:03::","synset_id":"study.v.02"},{"index":5,"sense_key":"study%2:31:00::","synset_id":"learn.v.04"},{"index":6,"sense_key":"study%2:31:02::","synset_id":"analyze.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"study%2:31:02::","call_ids":["semeval2015.d003.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"study%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":76},"cost":{"currency":"USD","total_usd":0.005615,"input_uncached_usd":0.0028850000000000004,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.5014829160645604}
{"item_id":"semeval2015.d003.s014.t001","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:10:01::","synset_id":"study.n.10"},{"index":3,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":4,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":5,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":6,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":7,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":8,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":9,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":10,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d003.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":280,"reasoning_output_tokens":265},"cost":{"currency":"USD","total_usd":0.011895000000000001,"input_uncached_usd":0.0034950000000000003,"input_cached_usd":0.0,"output_usd":0.0084,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":12.291317041963339}
{"item_id":"semeval2015.d003.s014.t002","gold_sense_keys":["carry_out%2:36:00::"],"candidates":[{"index":1,"sense_key":"carry_out%2:30:00::","synset_id":"follow_through.v.02"},{"index":2,"sense_key":"carry_out%2:36:00::","synset_id":"carry_through.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"carry_out%2:36:00::","call_ids":["semeval2015.d003.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"carry_out%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":101},"cost":{"currency":"USD","total_usd":0.005525,"input_uncached_usd":0.0020150000000000003,"input_cached_usd":0.0,"output_usd":0.00351,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2027966669993475}
{"item_id":"semeval2015.d003.s014.t003","gold_sense_keys":["laboratory%1:06:00::"],"candidates":[{"index":1,"sense_key":"laboratory%1:06:00::","synset_id":"lab.n.01"},{"index":2,"sense_key":"laboratory%1:15:00::","synset_id":"testing_ground.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"laboratory%1:06:00::","call_ids":["semeval2015.d003.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"laboratory%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":394,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":136},"cost":{"currency":"USD","total_usd":0.006500000000000001,"input_uncached_usd":0.00197,"input_cached_usd":0.0,"output_usd":0.00453,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.490581208025105}
{"item_id":"semeval2015.d003.s014.t004","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":2,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"},{"index":3,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":4,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":5,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":6,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":7,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"dog%1:05:00::","call_ids":["semeval2015.d003.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":40,"reasoning_output_tokens":25},"cost":{"currency":"USD","total_usd":0.004315,"input_uncached_usd":0.003115,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.987366999965161}
{"item_id":"semeval2015.d003.s014.t007","gold_sense_keys":["several%5:00:00:some:00"],"candidates":[{"index":1,"sense_key":"several%5:00:00:some:00","synset_id":"several.s.01"},{"index":2,"sense_key":"several%5:00:00:different:00","synset_id":"several.s.03"},{"index":3,"sense_key":"several%5:00:00:individual:00","synset_id":"respective.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"several%5:00:00:some:00","call_ids":["semeval2015.d003.s014.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"several%5:00:00:some:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":52},"cost":{"currency":"USD","total_usd":0.00412,"input_uncached_usd":0.0020800000000000003,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5651206669863313}
{"item_id":"semeval2015.d003.s015.t000","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["semeval2015.d003.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":380,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":57},"cost":{"currency":"USD","total_usd":0.00409,"input_uncached_usd":0.0019000000000000002,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.254034374956973}
{"item_id":"semeval2015.d003.s015.t001","gold_sense_keys":["show%2:32:00::","show%2:39:00::"],"candidates":[{"index":1,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":2,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":3,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":4,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":5,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":6,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":7,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":8,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":9,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":10,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":11,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":12,"sense_key":"show%2:33:00::","synset_id":"show.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"show%2:31:00::","call_ids":["semeval2015.d003.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"show%2:31:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":862,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02015,"input_uncached_usd":0.0043100000000000005,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.421351208002307}
{"item_id":"semeval2015.d003.s015.t002","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":3,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":4,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":5,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":9,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d003.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":706,"cached_input_tokens":0,"output_tokens":242,"reasoning_output_tokens":226},"cost":{"currency":"USD","total_usd":0.010790000000000001,"input_uncached_usd":0.00353,"input_cached_usd":0.0,"output_usd":0.00726,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.3939068340696394}
{"item_id":"semeval2015.d003.s016.t000","gold_sense_keys":["result%1:11:00::"],"candidates":[{"index":1,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":4,"sense_key":"result%1:11:00::","synset_id":"result.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"result%1:11:00::","call_ids":["semeval2015.d003.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"result%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":523,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018455,"input_uncached_usd":0.002615,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.038353458046913}
{"item_id":"semeval2015.d003.s016.t001","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":2,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":5,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":6,"sense_key":"study%1:10:01::","synset_id":"study.n.10"},{"index":7,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":8,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":9,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":10,"sense_key":"study%1:09:04::","synset_id":"study.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d003.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":458,"reasoning_output_tokens":442},"cost":{"currency":"USD","total_usd":0.01738,"input_uncached_usd":0.0036400000000000004,"input_cached_usd":0.0,"output_usd":0.01374,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.481975249946117}
{"item_id":"semeval2015.d003.s016.t002","gold_sense_keys":["show%2:31:00::","show%2:32:00::"],"candidates":[{"index":1,"sense_key":"show%2:33:00::","synset_id":"show.v.12"},{"index":2,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":3,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":4,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":5,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":6,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":7,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":8,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":9,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":10,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":11,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":12,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"show%2:32:00::","call_ids":["semeval2015.d003.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"show%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":884,"cached_input_tokens":0,"output_tokens":222,"reasoning_output_tokens":206},"cost":{"currency":"USD","total_usd":0.01108,"input_uncached_usd":0.00442,"input_cached_usd":0.0,"output_usd":0.00666,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.4082986660068855}
{"item_id":"semeval2015.d003.s016.t003","gold_sense_keys":["more%4:02:00::"],"candidates":[{"index":1,"sense_key":"more%4:02:00::","synset_id":"more.r.01"},{"index":2,"sense_key":"more%4:02:01::","synset_id":"more.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%4:02:00::","call_ids":["semeval2015.d003.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":398,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.00466,"input_uncached_usd":0.00199,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0490676660556346}
{"item_id":"semeval2015.d003.s016.t004","gold_sense_keys":["effective%3:00:00::","effective%5:00:00:competent:00","effective%5:00:00:efficacious:00"],"candidates":[{"index":1,"sense_key":"effective%3:00:00::","synset_id":"effective.a.01"},{"index":2,"sense_key":"effective%5:00:00:operational:00","synset_id":"effective.s.05"},{"index":3,"sense_key":"effective%5:00:00:actual:00","synset_id":"effective.s.04"},{"index":4,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":5,"sense_key":"effective%5:00:00:efficacious:00","synset_id":"effective.s.02"},{"index":6,"sense_key":"effective%5:00:00:competent:00","synset_id":"effective.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"effective%5:00:00:efficacious:00","call_ids":["semeval2015.d003.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"effective%5:00:00:efficacious:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":167,"reasoning_output_tokens":151},"cost":{"currency":"USD","total_usd":0.00818,"input_uncached_usd":0.00317,"input_cached_usd":0.0,"output_usd":0.00501,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":3.7953866659663618}
{"item_id":"semeval2015.d003.s016.t005","gold_sense_keys":["placebo%1:06:00::"],"candidates":[{"index":1,"sense_key":"placebo%1:06:00::","synset_id":"placebo.n.01"},{"index":2,"sense_key":"placebo%1:04:00::","synset_id":"placebo.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"placebo%1:06:00::","call_ids":["semeval2015.d003.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"placebo%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":28},"cost":{"currency":"USD","total_usd":0.003255,"input_uncached_usd":0.001935,"input_cached_usd":0.0,"output_usd":0.00132,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.2381264580180869}
{"item_id":"semeval2015.d003.s016.t006","gold_sense_keys":["less%3:00:00::"],"candidates":[{"index":1,"sense_key":"less%5:00:00:inferior:02","synset_id":"less.s.01"},{"index":2,"sense_key":"less%5:00:00:fewer:00","synset_id":"less.s.02"},{"index":3,"sense_key":"less%3:00:00::","synset_id":"less.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"less%3:00:00::","call_ids":["semeval2015.d003.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"less%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":110},"cost":{"currency":"USD","total_usd":0.00604,"input_uncached_usd":0.0022600000000000003,"input_cached_usd":0.0,"output_usd":0.00378,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.831217208993621}
{"item_id":"semeval2015.d003.s016.t008","gold_sense_keys":["see%2:39:02::"],"candidates":[{"index":1,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":2,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":3,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":4,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":5,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":6,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":7,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"},{"index":8,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":9,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":10,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":11,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":12,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":13,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":14,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":17,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":18,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":19,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":20,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":21,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":22,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":23,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":24,"sense_key":"see%2:39:05::","synset_id":"see.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"see%2:39:02::","call_ids":["semeval2015.d003.s016.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"see%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1457,"cached_input_tokens":0,"output_tokens":263,"reasoning_output_tokens":247},"cost":{"currency":"USD","total_usd":0.015175000000000001,"input_uncached_usd":0.007285000000000001,"input_cached_usd":0.0,"output_usd":0.00789,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":5.116162957972847}
{"item_id":"semeval2015.d003.s016.t009","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":2,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"},{"index":7,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dog%1:05:00::","call_ids":["semeval2015.d003.s016.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.00485,"input_uncached_usd":0.0032600000000000003,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.425881666946225}
{"item_id":"semeval2015.d003.s016.t011","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":2,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":3,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"},{"index":4,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d003.s016.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":36},"cost":{"currency":"USD","total_usd":0.00396,"input_uncached_usd":0.0024300000000000003,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.637566416990012}
{"item_id":"semeval2015.d003.s016.t012","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":2,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":3,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":4,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":5,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"dog%1:05:00::","call_ids":["semeval2015.d003.s016.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":31},"cost":{"currency":"USD","total_usd":0.0046700000000000005,"input_uncached_usd":0.0032600000000000003,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.385825917008333}
{"item_id":"semeval2015.d003.s016.t014","gold_sense_keys":["placebo%1:06:00::"],"candidates":[{"index":1,"sense_key":"placebo%1:04:00::","synset_id":"placebo.n.02"},{"index":2,"sense_key":"placebo%1:06:00::","synset_id":"placebo.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"placebo%1:06:00::","call_ids":["semeval2015.d003.s016.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"placebo%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":31,"reasoning_output_tokens":15},"cost":{"currency":"USD","total_usd":0.0028650000000000004,"input_uncached_usd":0.001935,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.26047541690059}
{"item_id":"semeval2015.d003.s016.t015","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":2,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s016.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":50},"cost":{"currency":"USD","total_usd":0.0044,"input_uncached_usd":0.0024500000000000004,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7100381669588387}
{"item_id":"semeval2015.d003.s017.t000","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":3,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"},{"index":4,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.004745,"input_uncached_usd":0.0024950000000000003,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.307723541976884}
{"item_id":"semeval2015.d003.s017.t002","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"},{"index":2,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":3,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":4,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":5,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"dog%1:05:00::","call_ids":["semeval2015.d003.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":40,"reasoning_output_tokens":25},"cost":{"currency":"USD","total_usd":0.004505,"input_uncached_usd":0.003305,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3993465419625863}
{"item_id":"semeval2015.d003.s017.t003","gold_sense_keys":["start%2:30:01::","start%2:41:04::"],"candidates":[{"index":1,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":2,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":3,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":4,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":5,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":6,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":7,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":8,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":9,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":10,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":11,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":12,"sense_key":"start%2:38:03::","synset_id":"start.v.14"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"start%2:30:01::","call_ids":["semeval2015.d003.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"start%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":949,"cached_input_tokens":0,"output_tokens":799,"reasoning_output_tokens":777},"cost":{"currency":"USD","total_usd":0.028715,"input_uncached_usd":0.004745,"input_cached_usd":0.0,"output_usd":0.02397,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":15.345821292023174}
{"item_id":"semeval2015.d003.s017.t004","gold_sense_keys":["injection%1:04:01::"],"candidates":[{"index":1,"sense_key":"injection%1:04:01::","synset_id":"injection.n.03"},{"index":2,"sense_key":"injection%1:04:00::","synset_id":"injection.n.01"},{"index":3,"sense_key":"injection%1:27:00::","synset_id":"injection.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"injection%1:27:00::","call_ids":["semeval2015.d003.s017.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"injection%1:27:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":377,"reasoning_output_tokens":361},"cost":{"currency":"USD","total_usd":0.01344,"input_uncached_usd":0.0021300000000000004,"input_cached_usd":0.0,"output_usd":0.01131,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":6.541985083953477}
{"item_id":"semeval2015.d003.s017.t008","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"},{"index":2,"sense_key":"able%3:00:00::","synset_id":"able.a.01"},{"index":3,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"},{"index":4,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"able%3:00:00::","call_ids":["semeval2015.d003.s017.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"able%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":508,"reasoning_output_tokens":492},"cost":{"currency":"USD","total_usd":0.017785000000000002,"input_uncached_usd":0.0025450000000000004,"input_cached_usd":0.0,"output_usd":0.01524,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.028811666998081}
{"item_id":"semeval2015.d003.s017.t009","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"},{"index":2,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"},{"index":3,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"},{"index":4,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s017.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.00409,"input_uncached_usd":0.0022600000000000003,"input_cached_usd":0.0,"output_usd":0.00183,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5826543750008568}
{"item_id":"semeval2015.d003.s017.t010","gold_sense_keys":["intact%5:00:00:undamaged:00","intact%5:00:00:whole:00"],"candidates":[{"index":1,"sense_key":"intact%5:00:00:uncastrated:00","synset_id":"entire.s.04"},{"index":2,"sense_key":"intact%5:00:00:uninjured:00","synset_id":"intact.s.02"},{"index":3,"sense_key":"intact%5:00:00:undamaged:00","synset_id":"intact.s.04"},{"index":4,"sense_key":"intact%5:00:00:whole:00","synset_id":"integral.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"intact%5:00:00:whole:00","call_ids":["semeval2015.d003.s017.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"intact%5:00:00:whole:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01839,"input_uncached_usd":0.00255,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.455916875042021}
{"item_id":"semeval2015.d003.s017.t011","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"},{"index":2,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"},{"index":3,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"},{"index":4,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s017.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":43},"cost":{"currency":"USD","total_usd":0.004030000000000001,"input_uncached_usd":0.0022600000000000003,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8898974169278517}
{"item_id":"semeval2015.d003.s018.t000","gold_sense_keys":["follow-up%1:04:00::","follow-up%1:04:01::"],"candidates":[{"index":1,"sense_key":"follow-up%1:06:00::","synset_id":"follow-up.n.01"},{"index":2,"sense_key":"follow-up%1:04:01::","synset_id":"follow-up.n.02"},{"index":3,"sense_key":"follow-up%1:04:00::","synset_id":"follow-up.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"follow-up%1:04:01::","call_ids":["semeval2015.d003.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"follow-up%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":74},"cost":{"currency":"USD","total_usd":0.004920000000000001,"input_uncached_usd":0.00222,"input_cached_usd":0.0,"output_usd":0.0027,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7790117919212207}
{"item_id":"semeval2015.d003.s018.t001","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":62},"cost":{"currency":"USD","total_usd":0.004815,"input_uncached_usd":0.0025050000000000003,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.343899416970089}
{"item_id":"semeval2015.d003.s018.t002","gold_sense_keys":["owner%1:18:02::"],"candidates":[{"index":1,"sense_key":"owner%1:18:00::","synset_id":"owner.n.01"},{"index":2,"sense_key":"owner%1:18:02::","synset_id":"owner.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"owner%1:18:02::","call_ids":["semeval2015.d003.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"owner%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":421,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":56},"cost":{"currency":"USD","total_usd":0.004235,"input_uncached_usd":0.002105,"input_cached_usd":0.0,"output_usd":0.00213,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0767579169478267}
{"item_id":"semeval2015.d003.s018.t003","gold_sense_keys":["administer%2:29:00::"],"candidates":[{"index":1,"sense_key":"administer%2:29:00::","synset_id":"administer.v.04"},{"index":2,"sense_key":"administer%2:40:01::","synset_id":"administer.v.02"},{"index":3,"sense_key":"administer%2:40:00::","synset_id":"distribute.v.01"},{"index":4,"sense_key":"administer%2:41:01::","synset_id":"administer.v.05"},{"index":5,"sense_key":"administer%2:41:00::","synset_id":"administer.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"administer%2:29:00::","call_ids":["semeval2015.d003.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"administer%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":51},"cost":{"currency":"USD","total_usd":0.004675,"input_uncached_usd":0.0026650000000000003,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.631150458008051}
{"item_id":"semeval2015.d003.s018.t004","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"},{"index":2,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"},{"index":3,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"},{"index":4,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":30,"reasoning_output_tokens":14},"cost":{"currency":"USD","total_usd":0.0031800000000000005,"input_uncached_usd":0.0022800000000000003,"input_cached_usd":0.0,"output_usd":0.0009,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.3562556670513004}
{"item_id":"semeval2015.d003.s018.t005","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":2,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":3,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"},{"index":6,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":7,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dog%1:05:00::","call_ids":["semeval2015.d003.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":30},"cost":{"currency":"USD","total_usd":0.004695,"input_uncached_usd":0.0033150000000000002,"input_cached_usd":0.0,"output_usd":0.00138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.5797278750687838}
{"item_id":"semeval2015.d003.s019.t001","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":67},"cost":{"currency":"USD","total_usd":0.00498,"input_uncached_usd":0.00252,"input_cached_usd":0.0,"output_usd":0.00246,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.503795790951699}
{"item_id":"semeval2015.d003.s019.t003","gold_sense_keys":["only%4:02:01::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":3,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":4,"sense_key":"only%4:02:05::","synset_id":"only.r.07"},{"index":5,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":6,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":7,"sense_key":"only%4:02:00::","synset_id":"only.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"only%4:02:03::","call_ids":["semeval2015.d003.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"only%4:02:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018875,"input_uncached_usd":0.0030350000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.033186250017025}
{"item_id":"semeval2015.d003.s019.t005","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":2,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":3,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":4,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"other%3:00:00::","call_ids":["semeval2015.d003.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":44},"cost":{"currency":"USD","total_usd":0.004275,"input_uncached_usd":0.0025050000000000003,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7862292909994721}
{"item_id":"semeval2015.d003.s019.t007","gold_sense_keys":["measure%1:04:01::"],"candidates":[{"index":1,"sense_key":"measure%1:10:02::","synset_id":"bill.n.01"},{"index":2,"sense_key":"measure%1:06:01::","synset_id":"measure.n.09"},{"index":3,"sense_key":"measure%1:10:01::","synset_id":"meter.n.03"},{"index":4,"sense_key":"measure%1:10:03::","synset_id":"standard.n.01"},{"index":5,"sense_key":"measure%1:10:00::","synset_id":"measure.n.07"},{"index":6,"sense_key":"measure%1:06:00::","synset_id":"measuring_stick.n.01"},{"index":7,"sense_key":"measure%1:04:00::","synset_id":"measurement.n.01"},{"index":8,"sense_key":"measure%1:03:00::","synset_id":"measure.n.02"},{"index":9,"sense_key":"measure%1:04:01::","synset_id":"measure.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"measure%1:04:01::","call_ids":["semeval2015.d003.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"measure%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":726,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.0066300000000000005,"input_uncached_usd":0.0036300000000000004,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.084920290973969}
{"item_id":"semeval2015.d003.s019.t008","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"},{"index":2,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":3,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":4,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"other%3:00:00::","call_ids":["semeval2015.d003.s019.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":46},"cost":{"currency":"USD","total_usd":0.004365,"input_uncached_usd":0.0025050000000000003,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.598638917086646}
{"item_id":"semeval2015.d003.s019.t011","gold_sense_keys":["address%2:34:00::"],"candidates":[{"index":1,"sense_key":"address%2:40:00::","synset_id":"address.v.07"},{"index":2,"sense_key":"address%2:32:00::","synset_id":"address.v.01"},{"index":3,"sense_key":"address%2:32:04::","synset_id":"cover.v.05"},{"index":4,"sense_key":"address%2:41:08::","synset_id":"address.v.06"},{"index":5,"sense_key":"address%2:30:00::","synset_id":"address.v.10"},{"index":6,"sense_key":"address%2:33:00::","synset_id":"address.v.04"},{"index":7,"sense_key":"address%2:32:01::","synset_id":"address.v.09"},{"index":8,"sense_key":"address%2:32:02::","synset_id":"address.v.02"},{"index":9,"sense_key":"address%2:32:03::","synset_id":"address.v.03"},{"index":10,"sense_key":"address%2:34:00::","synset_id":"address.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"address%2:34:00::","call_ids":["semeval2015.d003.s019.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"address%2:34:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":183,"reasoning_output_tokens":167},"cost":{"currency":"USD","total_usd":0.00882,"input_uncached_usd":0.00333,"input_cached_usd":0.0,"output_usd":0.00549,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":4.114495875081047}
{"item_id":"semeval2015.d003.s019.t012","gold_sense_keys":["underlying%5:00:00:implicit:00"],"candidates":[{"index":1,"sense_key":"underlying%5:00:00:subjacent:00","synset_id":"underlying.s.02"},{"index":2,"sense_key":"underlying%5:00:00:implicit:00","synset_id":"implicit_in.s.01"},{"index":3,"sense_key":"underlying%5:00:00:basic:00","synset_id":"fundamental.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"underlying%5:00:00:basic:00","call_ids":["semeval2015.d003.s019.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"underlying%5:00:00:basic:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.033665,"input_uncached_usd":0.002285,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":17.255717583931983}
{"item_id":"semeval2015.d003.s019.t013","gold_sense_keys":["cause%1:10:00::","cause%1:11:00::"],"candidates":[{"index":1,"sense_key":"cause%1:04:00::","synset_id":"lawsuit.n.01"},{"index":2,"sense_key":"cause%1:04:01::","synset_id":"campaign.n.02"},{"index":3,"sense_key":"cause%1:10:00::","synset_id":"cause.n.02"},{"index":4,"sense_key":"cause%1:11:00::","synset_id":"cause.n.01"},{"index":5,"sense_key":"cause%1:03:00::","synset_id":"causal_agent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"cause%1:11:00::","call_ids":["semeval2015.d003.s019.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"cause%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":512,"reasoning_output_tokens":496},"cost":{"currency":"USD","total_usd":0.018205,"input_uncached_usd":0.0028450000000000003,"input_cached_usd":0.0,"output_usd":0.01536,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.344250624999404}
{"item_id":"semeval2015.d003.s020.t000","gold_sense_keys":["risk%1:26:00::"],"candidates":[{"index":1,"sense_key":"risk%1:07:02::","synset_id":"risk.n.03"},{"index":2,"sense_key":"risk%1:26:00::","synset_id":"hazard.n.01"},{"index":3,"sense_key":"risk%1:04:00::","synset_id":"risk.n.02"},{"index":4,"sense_key":"risk%1:07:01::","synset_id":"risk.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"risk%1:26:00::","call_ids":["semeval2015.d003.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"risk%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.0049700000000000005,"input_uncached_usd":0.0025700000000000002,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8896475409856066}
{"item_id":"semeval2015.d003.s021.t000","gold_sense_keys":["generally%4:02:02::"],"candidates":[{"index":1,"sense_key":"generally%4:02:01::","synset_id":"generally.r.02"},{"index":2,"sense_key":"generally%4:02:02::","synset_id":"by_and_large.r.01"},{"index":3,"sense_key":"generally%4:02:00::","synset_id":"broadly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"generally%4:02:02::","call_ids":["semeval2015.d003.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"generally%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.01822,"input_uncached_usd":0.00238,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.41382233297918}
{"item_id":"semeval2015.d003.s021.t001","gold_sense_keys":["well%4:02:00::"],"candidates":[{"index":1,"sense_key":"well%4:02:07::","synset_id":"well.r.06"},{"index":2,"sense_key":"well%4:02:12::","synset_id":"well.r.10"},{"index":3,"sense_key":"well%4:02:10::","synset_id":"well.r.11"},{"index":4,"sense_key":"well%4:02:08::","synset_id":"well.r.05"},{"index":5,"sense_key":"well%4:02:11::","synset_id":"well.r.07"},{"index":6,"sense_key":"well%4:02:14::","synset_id":"well.r.08"},{"index":7,"sense_key":"well%4:02:15::","synset_id":"well.r.04"},{"index":8,"sense_key":"well%4:02:03::","synset_id":"well.r.03"},{"index":9,"sense_key":"well%4:02:13::","synset_id":"well.r.09"},{"index":10,"sense_key":"well%4:02:00::","synset_id":"well.r.01"},{"index":11,"sense_key":"well%4:02:01::","synset_id":"well.r.13"},{"index":12,"sense_key":"well%4:02:05::","synset_id":"well.r.02"},{"index":13,"sense_key":"well%4:02:09::","synset_id":"well.r.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"well%4:02:00::","call_ids":["semeval2015.d003.s021.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"well%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":916,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.02042,"input_uncached_usd":0.004580000000000001,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":10.71985495800618}
{"item_id":"semeval2015.d003.s021.t002","gold_sense_keys":["tolerate%2:39:00::"],"candidates":[{"index":1,"sense_key":"tolerate%2:41:01::","synset_id":"tolerate.v.02"},{"index":2,"sense_key":"tolerate%2:32:01::","synset_id":"allow.v.10"},{"index":3,"sense_key":"tolerate%2:31:00::","synset_id":"digest.v.03"},{"index":4,"sense_key":"tolerate%2:39:00::","synset_id":"tolerate.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tolerate%2:39:00::","call_ids":["semeval2015.d003.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tolerate%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":53},"cost":{"currency":"USD","total_usd":0.004805,"input_uncached_usd":0.002735,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7330732910195366}
{"item_id":"semeval2015.d003.s021.t003","gold_sense_keys":["administer%2:29:00::"],"candidates":[{"index":1,"sense_key":"administer%2:41:00::","synset_id":"administer.v.01"},{"index":2,"sense_key":"administer%2:41:01::","synset_id":"administer.v.05"},{"index":3,"sense_key":"administer%2:40:01::","synset_id":"administer.v.02"},{"index":4,"sense_key":"administer%2:40:00::","synset_id":"distribute.v.01"},{"index":5,"sense_key":"administer%2:29:00::","synset_id":"administer.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"administer%2:29:00::","call_ids":["semeval2015.d003.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"administer%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.004375,"input_uncached_usd":0.002755,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.9349499579984695}
{"item_id":"semeval2015.d003.s021.t004","gold_sense_keys":["daily%4:02:00::"],"candidates":[{"index":1,"sense_key":"daily%4:02:01::","synset_id":"day_by_day.r.01"},{"index":2,"sense_key":"daily%4:02:00::","synset_id":"daily.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"daily%4:02:00::","call_ids":["semeval2015.d003.s021.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"daily%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":41},"cost":{"currency":"USD","total_usd":0.0037250000000000004,"input_uncached_usd":0.0020450000000000004,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.0435452919919044}
{"item_id":"semeval2015.d003.s021.t005","gold_sense_keys":["dosage%1:06:00::"],"candidates":[{"index":1,"sense_key":"dosage%1:23:00::","synset_id":"dose.n.02"},{"index":2,"sense_key":"dosage%1:06:00::","synset_id":"dose.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dosage%1:23:00::","call_ids":["semeval2015.d003.s021.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dosage%1:23:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":527,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.017890000000000003,"input_uncached_usd":0.0020800000000000003,"input_cached_usd":0.0,"output_usd":0.01581,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":9.07057308300864}
{"item_id":"semeval2015.d003.s021.t008","gold_sense_keys":["dose%1:06:00::"],"candidates":[{"index":1,"sense_key":"dose%1:06:00::","synset_id":"dose.n.01"},{"index":2,"sense_key":"dose%1:06:01::","synset_id":"acid.n.02"},{"index":3,"sense_key":"dose%1:23:00::","synset_id":"dose.n.02"},{"index":4,"sense_key":"dose%1:26:00::","synset_id":"venereal_disease.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dose%1:06:00::","call_ids":["semeval2015.d003.s021.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dose%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":480,"reasoning_output_tokens":464},"cost":{"currency":"USD","total_usd":0.016965,"input_uncached_usd":0.002565,"input_cached_usd":0.0,"output_usd":0.0144,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":7.704365749959834}
{"item_id":"semeval2015.d003.s021.t009","gold_sense_keys":["extended%5:00:00:long:02"],"candidates":[{"index":1,"sense_key":"extended%5:00:00:figurative:00","synset_id":"extended.s.03"},{"index":2,"sense_key":"extended%3:00:00::","synset_id":"extended.a.02"},{"index":3,"sense_key":"extended%5:00:00:long:02","synset_id":"drawn-out.s.01"},{"index":4,"sense_key":"extended%5:00:00:large:00","synset_id":"extensive.s.01"},{"index":5,"sense_key":"extended%5:00:00:long:01","synset_id":"elongated.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"extended%5:00:00:long:02","call_ids":["semeval2015.d003.s021.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"extended%5:00:00:long:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":581,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":38},"cost":{"currency":"USD","total_usd":0.004495,"input_uncached_usd":0.002905,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.647684375056997}
{"item_id":"semeval2015.d003.s021.t013","gold_sense_keys":["duration%1:07:00::","duration%1:28:02::"],"candidates":[{"index":1,"sense_key":"duration%1:07:00::","synset_id":"duration.n.03"},{"index":2,"sense_key":"duration%1:28:02::","synset_id":"duration.n.01"},{"index":3,"sense_key":"duration%1:28:00::","synset_id":"duration.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"duration%1:07:00::","call_ids":["semeval2015.d003.s021.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"duration%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":767,"reasoning_output_tokens":745},"cost":{"currency":"USD","total_usd":0.025235000000000004,"input_uncached_usd":0.002225,"input_cached_usd":0.0,"output_usd":0.02301,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":11.695356542011723}
{"item_id":"semeval2015.d003.s021.t014","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"},{"index":3,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":4,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s021.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":84},"cost":{"currency":"USD","total_usd":0.00559,"input_uncached_usd":0.0025900000000000003,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.051753708976321}
{"item_id":"semeval2015.d003.s022.t000","gold_sense_keys":["precaution%1:04:00::"],"candidates":[{"index":1,"sense_key":"precaution%1:07:00::","synset_id":"precaution.n.02"},{"index":2,"sense_key":"precaution%1:09:00::","synset_id":"caution.n.03"},{"index":3,"sense_key":"precaution%1:04:00::","synset_id":"precaution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"precaution%1:04:00::","call_ids":["semeval2015.d003.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"precaution%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":102},"cost":{"currency":"USD","total_usd":0.00578,"input_uncached_usd":0.0022400000000000002,"input_cached_usd":0.0,"output_usd":0.00354,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.2065847909543663}
{"item_id":"semeval2015.d003.s022.t001","gold_sense_keys":["person%1:03:00::"],"candidates":[{"index":1,"sense_key":"person%1:08:00::","synset_id":"person.n.02"},{"index":2,"sense_key":"person%1:10:00::","synset_id":"person.n.03"},{"index":3,"sense_key":"person%1:03:00::","synset_id":"person.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"person%1:03:00::","call_ids":["semeval2015.d003.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"person%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":37},"cost":{"currency":"USD","total_usd":0.00383,"input_uncached_usd":0.0022400000000000002,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8034363329643384}
{"item_id":"semeval2015.d003.s022.t002","gold_sense_keys":["give%2:40:13::"],"candidates":[{"index":1,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":2,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":5,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":6,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":7,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":8,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":9,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":10,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":11,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":12,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":13,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":14,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":15,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":16,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":17,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":18,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":19,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":20,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":21,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":22,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":23,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":24,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":25,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":26,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":27,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":28,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":29,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":30,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":31,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":32,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":33,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":34,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":35,"sense_key":"give%2:29:10::","synset_id":"give.v.44"},{"index":36,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":37,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":38,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":39,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":40,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":41,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":42,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":43,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":44,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":37,"chosen_sense_key":"give%2:40:13::","call_ids":["semeval2015.d003.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":37,"predicted_sense_key":"give%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2058,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":78},"cost":{"currency":"USD","total_usd":0.013080000000000001,"input_uncached_usd":0.01029,"input_cached_usd":0.0,"output_usd":0.00279,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.724833165993914}
{"item_id":"semeval2015.d003.s022.t003","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":2,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":3,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"},{"index":4,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d003.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":34},"cost":{"currency":"USD","total_usd":0.003905,"input_uncached_usd":0.002405,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.7190316249616444}
{"item_id":"semeval2015.d003.s022.t005","gold_sense_keys":["contact%1:04:00::","contact%1:26:00::"],"candidates":[{"index":1,"sense_key":"contact%1:06:00::","synset_id":"contact.n.07"},{"index":2,"sense_key":"contact%1:10:00::","synset_id":"liaison.n.02"},{"index":3,"sense_key":"contact%1:06:01::","synset_id":"contact.n.09"},{"index":4,"sense_key":"contact%1:26:00::","synset_id":"contact.n.03"},{"index":5,"sense_key":"contact%1:11:00::","synset_id":"contact.n.04"},{"index":6,"sense_key":"contact%1:10:01::","synset_id":"contact.n.08"},{"index":7,"sense_key":"contact%1:04:00::","synset_id":"contact.n.02"},{"index":8,"sense_key":"contact%1:18:00::","synset_id":"contact.n.05"},{"index":9,"sense_key":"contact%1:04:02::","synset_id":"contact.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"contact%1:04:00::","call_ids":["semeval2015.d003.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"contact%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":1046,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.03489,"input_uncached_usd":0.00351,"input_cached_usd":0.0,"output_usd":0.03138,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":19.82110195804853}
{"item_id":"semeval2015.d003.s023.t001","gold_sense_keys":["represent%2:42:00::"],"candidates":[{"index":1,"sense_key":"represent%2:32:10::","synset_id":"represent.v.13"},{"index":2,"sense_key":"represent%2:41:00::","synset_id":"represent.v.04"},{"index":3,"sense_key":"represent%2:32:11::","synset_id":"present.v.02"},{"index":4,"sense_key":"represent%2:42:01::","synset_id":"exemplify.v.01"},{"index":5,"sense_key":"represent%2:42:02::","synset_id":"represent.v.01"},{"index":6,"sense_key":"represent%2:32:12::","synset_id":"represent.v.12"},{"index":7,"sense_key":"represent%2:36:02::","synset_id":"act.v.03"},{"index":8,"sense_key":"represent%2:36:01::","synset_id":"represent.v.09"},{"index":9,"sense_key":"represent%2:30:06::","synset_id":"map.v.06"},{"index":10,"sense_key":"represent%2:41:02::","synset_id":"represent.v.03"},{"index":11,"sense_key":"represent%2:32:00::","synset_id":"represent.v.05"},{"index":12,"sense_key":"represent%2:42:00::","synset_id":"constitute.v.01"},{"index":13,"sense_key":"represent%2:41:06::","synset_id":"defend.v.06"},{"index":14,"sense_key":"represent%2:36:11::","synset_id":"stage.v.01"},{"index":15,"sense_key":"represent%2:32:02::","synset_id":"typify.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"represent%2:42:00::","call_ids":["semeval2015.d003.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"represent%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":953,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":122},"cost":{"currency":"USD","total_usd":0.008905,"input_uncached_usd":0.004765,"input_cached_usd":0.0,"output_usd":0.0041400000000000005,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":2.805855834041722}
{"item_id":"semeval2015.d003.s023.t002","gold_sense_keys":["particular%5:00:00:uncommon:00","particular%5:00:02:specific:00"],"candidates":[{"index":1,"sense_key":"particular%5:00:00:uncommon:00","synset_id":"especial.s.01"},{"index":2,"sense_key":"particular%5:00:02:specific:00","synset_id":"particular.s.02"},{"index":3,"sense_key":"particular%5:00:00:primary:00","synset_id":"particular.s.04"},{"index":4,"sense_key":"particular%5:00:00:fastidious:00","synset_id":"finical.s.01"},{"index":5,"sense_key":"particular%5:00:00:specific:00","synset_id":"particular.s.01"},{"index":6,"sense_key":"particular%5:00:00:careful:00","synset_id":"particular.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"particular%5:00:00:specific:00","call_ids":["semeval2015.d003.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"particular%5:00:00:specific:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.018705,"input_uncached_usd":0.0028650000000000004,"input_cached_usd":0.0,"output_usd":0.01584,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":8.078761499957182}
{"item_id":"semeval2015.d003.s023.t003","gold_sense_keys":["risk%1:26:00::"],"candidates":[{"index":1,"sense_key":"risk%1:26:00::","synset_id":"hazard.n.01"},{"index":2,"sense_key":"risk%1:07:02::","synset_id":"risk.n.03"},{"index":3,"sense_key":"risk%1:07:01::","synset_id":"risk.n.04"},{"index":4,"sense_key":"risk%1:04:00::","synset_id":"risk.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"risk%1:26:00::","call_ids":["semeval2015.d003.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"risk%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":54},"cost":{"currency":"USD","total_usd":0.0043300000000000005,"input_uncached_usd":0.0022600000000000003,"input_cached_usd":0.0,"output_usd":0.0020700000000000002,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.6892321249470115}
{"item_id":"semeval2015.d003.s023.t004","gold_sense_keys":["people%1:14:00::","people%1:14:03::"],"candidates":[{"index":1,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":2,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":3,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"people%1:14:00::","call_ids":["semeval2015.d003.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":60},"cost":{"currency":"USD","total_usd":0.00453,"input_uncached_usd":0.0022500000000000003,"input_cached_usd":0.0,"output_usd":0.00228,"input_uncached_unit_price_usd":5e-6,"input_cached_unit_price_usd":5e-7,"output_unit_price_usd":0.00003,"source":"litellm_estimate"},"latency_seconds":1.8190820410382003}